2024-03-26T20:39:10.944 INFO:root:teuthology version: 0.0.1.dev262+ge691533 2024-03-26T20:39:10.944 DEBUG:teuthology.run:Teuthology command: teuthology --description orch:cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} --verbose --name gabrioux-2024-03-26_20:31:35-orch:cephadm-wip-guits-testing-2024-03-26-0925-distro-default-smithi --archive /home/teuthworker/archive/gabrioux-2024-03-26_20:31:35-orch:cephadm-wip-guits-testing-2024-03-26-0925-distro-default-smithi/7624079 --owner scheduled_gabrioux@teuthology -- /home/teuthworker/archive/gabrioux-2024-03-26_20:31:35-orch:cephadm-wip-guits-testing-2024-03-26-0925-distro-default-smithi/7624079/orig.config.yaml 2024-03-26T20:39:10.983 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-03-26T20:39:11.069 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/gabrioux-2024-03-26_20:31:35-orch:cephadm-wip-guits-testing-2024-03-26-0925-distro-default-smithi/7624079 branch: wip-guits-testing-2024-03-26-0925 description: orch:cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} email: gabrioux@ibm.com first_in_suite: false job_id: '7624079' kernel: kdb: true sha1: distro last_in_suite: false machine_type: smithi name: gabrioux-2024-03-26_20:31:35-orch:cephadm-wip-guits-testing-2024-03-26-0925-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 9.stream overrides: admin_socket: branch: wip-guits-testing-2024-03-26-0925 ceph: conf: mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: 42fc52b86b14d4f88c40d75eb7102e0ab3918f55 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: 42fc52b86b14d4f88c40d75eb7102e0ab3918f55 selinux: allowlist: - scontext=system_u:system_r:logrotate_t:s0 - scontext=system_u:system_r:getty_t:s0 workunit: branch: wip-guits-testing-2024-03-26-0925 sha1: 42fc52b86b14d4f88c40d75eb7102e0ab3918f55 owner: scheduled_gabrioux@teuthology priority: 70 repo: https://git.ceph.com/ceph-ci.git roles: - - host.a - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - - host.b - osd.4 - osd.5 - osd.6 - osd.7 seed: 832 sha1: 42fc52b86b14d4f88c40d75eb7102e0ab3918f55 sleep_before_teardown: 0 subset: 1/8 suite: orch:cephadm suite_branch: wip-guits-testing-2024-03-26-0925 suite_path: /home/teuthworker/src/git.ceph.com_ceph-c_42fc52b86b14d4f88c40d75eb7102e0ab3918f55/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph-ci.git suite_sha1: 42fc52b86b14d4f88c40d75eb7102e0ab3918f55 targets: smithi063.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBC9Hv14bD1Z0Q7RqOa2RYZQs2kYcksnP86pj5bqMVjbFB2k7PsJJwRxCDYCetJe6lOxb1+jyC8DTngpKTqvZILM= smithi174.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBHAyroYl6r4dC7FKie8aWY8wnEkQIOSzm4Ncr41IYmxEaxKxyLwCpHcvF8/9ahDJrrhVKyEhgnaphxd67ZrFGgg= tasks: - cephadm: cephadm_branch: v17.2.0 cephadm_git_url: https://github.com/ceph/ceph image: quay.io/ceph/ceph:v17.2.0 roleless: true - cephadm.shell: host.a: - ceph orch status - ceph orch ps - ceph orch ls - ceph orch host ls - ceph orch device ls - vip.exec: all-hosts: - systemctl stop nfs-server - cephadm.shell: host.a: - ceph fs volume create foofs - cephadm.wait_for_service: service: mds.foofs - cephadm.shell: host.a: - ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2 - ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake - while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done - vip.exec: host.a: - mkdir /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done - echo test > /mnt/foo/testfile - sync - parallel: - upgrade-tasks - workload-tasks - vip.exec: host.a: - umount /mnt/foo - cephadm.shell: host.a: - ceph nfs cluster ls | grep foo - ceph nfs export ls foo --detailed - rados -p .nfs --all ls - - cephadm.shell: host.a: - 'set -ex [[ `ceph config get mgr mgr/cephadm/migration_current` -gt 2 ]] ' teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: e691533f9cbb33d85b2187bba20d7102f098636d timestamp: 2024-03-26_20:31:35 tube: smithi upgrade-tasks: sequential: - cephadm.shell: env: - sha1 host.a: - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph config set global log_to_journald false --force - ceph mgr module enable nfs --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - cephadm.shell: env: - sha1 host.a: - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done - ceph orch ps - ceph versions - echo "wait for servicemap items w/ changing names to refresh" - sleep 60 - ceph orch ps - ceph orch upgrade status - ceph health detail - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 - cephadm.wait_for_service: service: nfs.foo user: gabrioux verbose: true worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2226885 workload-tasks: sequential: - exec: host.a: - cd /mnt/foo && dbench 5 -t 600 || true - umount /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo ; do sleep 5 ; done - cd /mnt/foo && dbench 5 -t 5 2024-03-26T20:39:11.070 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph-c_42fc52b86b14d4f88c40d75eb7102e0ab3918f55/qa; will attempt to use it 2024-03-26T20:39:11.070 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph-c_42fc52b86b14d4f88c40d75eb7102e0ab3918f55/qa/tasks 2024-03-26T20:39:11.071 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-03-26T20:39:11.072 INFO:teuthology.task.internal:Checking packages... 2024-03-26T20:39:11.097 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash '42fc52b86b14d4f88c40d75eb7102e0ab3918f55' 2024-03-26T20:39:11.098 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-03-26T20:39:11.098 INFO:teuthology.packaging:ref: None 2024-03-26T20:39:11.098 INFO:teuthology.packaging:tag: None 2024-03-26T20:39:11.098 INFO:teuthology.packaging:branch: wip-guits-testing-2024-03-26-0925 2024-03-26T20:39:11.098 INFO:teuthology.packaging:sha1: 42fc52b86b14d4f88c40d75eb7102e0ab3918f55 2024-03-26T20:39:11.098 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=wip-guits-testing-2024-03-26-0925 2024-03-26T20:39:11.246 INFO:teuthology.task.internal:Found packages for ceph version 19.0.0-2484.g42fc52b8 2024-03-26T20:39:11.248 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-03-26T20:39:11.254 INFO:teuthology.task.internal:no buildpackages task found 2024-03-26T20:39:11.254 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-03-26T20:39:11.262 INFO:teuthology.task.internal:Saving configuration 2024-03-26T20:39:11.277 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-03-26T20:39:11.282 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-03-26T20:39:11.309 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi063.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/gabrioux-2024-03-26_20:31:35-orch:cephadm-wip-guits-testing-2024-03-26-0925-distro-default-smithi/7624079', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-03-26 20:33:58.548966', 'locked_by': 'scheduled_gabrioux@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBC9Hv14bD1Z0Q7RqOa2RYZQs2kYcksnP86pj5bqMVjbFB2k7PsJJwRxCDYCetJe6lOxb1+jyC8DTngpKTqvZILM='} 2024-03-26T20:39:11.334 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi174.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/gabrioux-2024-03-26_20:31:35-orch:cephadm-wip-guits-testing-2024-03-26-0925-distro-default-smithi/7624079', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-03-26 20:33:58.550526', 'locked_by': 'scheduled_gabrioux@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBHAyroYl6r4dC7FKie8aWY8wnEkQIOSzm4Ncr41IYmxEaxKxyLwCpHcvF8/9ahDJrrhVKyEhgnaphxd67ZrFGgg='} 2024-03-26T20:39:11.335 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-03-26T20:39:11.341 INFO:teuthology.task.internal:roles: ubuntu@smithi063.front.sepia.ceph.com - ['host.a', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0'] 2024-03-26T20:39:11.341 INFO:teuthology.task.internal:roles: ubuntu@smithi174.front.sepia.ceph.com - ['host.b', 'osd.4', 'osd.5', 'osd.6', 'osd.7'] 2024-03-26T20:39:11.341 INFO:teuthology.run_tasks:Running task console_log... 2024-03-26T20:39:11.411 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7fd4e3a2af70>, signals=[15]) 2024-03-26T20:39:11.412 INFO:teuthology.run_tasks:Running task internal.connect... 2024-03-26T20:39:11.417 INFO:teuthology.task.internal:Opening connections... 2024-03-26T20:39:11.417 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi063.front.sepia.ceph.com 2024-03-26T20:39:11.419 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi063.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-26T20:39:11.527 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi174.front.sepia.ceph.com 2024-03-26T20:39:11.528 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi174.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-26T20:39:11.616 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-03-26T20:39:11.623 DEBUG:teuthology.orchestra.run.smithi063:> uname -m 2024-03-26T20:39:11.641 INFO:teuthology.orchestra.run.smithi063.stdout:x86_64 2024-03-26T20:39:11.641 DEBUG:teuthology.orchestra.run.smithi063:> cat /etc/os-release 2024-03-26T20:39:11.699 INFO:teuthology.orchestra.run.smithi063.stdout:NAME="CentOS Stream" 2024-03-26T20:39:11.700 INFO:teuthology.orchestra.run.smithi063.stdout:VERSION="9" 2024-03-26T20:39:11.700 INFO:teuthology.orchestra.run.smithi063.stdout:ID="centos" 2024-03-26T20:39:11.700 INFO:teuthology.orchestra.run.smithi063.stdout:ID_LIKE="rhel fedora" 2024-03-26T20:39:11.700 INFO:teuthology.orchestra.run.smithi063.stdout:VERSION_ID="9" 2024-03-26T20:39:11.700 INFO:teuthology.orchestra.run.smithi063.stdout:PLATFORM_ID="platform:el9" 2024-03-26T20:39:11.700 INFO:teuthology.orchestra.run.smithi063.stdout:PRETTY_NAME="CentOS Stream 9" 2024-03-26T20:39:11.700 INFO:teuthology.orchestra.run.smithi063.stdout:ANSI_COLOR="0;31" 2024-03-26T20:39:11.700 INFO:teuthology.orchestra.run.smithi063.stdout:LOGO="fedora-logo-icon" 2024-03-26T20:39:11.700 INFO:teuthology.orchestra.run.smithi063.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-03-26T20:39:11.700 INFO:teuthology.orchestra.run.smithi063.stdout:HOME_URL="https://centos.org/" 2024-03-26T20:39:11.700 INFO:teuthology.orchestra.run.smithi063.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-03-26T20:39:11.700 INFO:teuthology.orchestra.run.smithi063.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-03-26T20:39:11.700 INFO:teuthology.orchestra.run.smithi063.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-03-26T20:39:11.701 INFO:teuthology.lock.ops:Updating smithi063.front.sepia.ceph.com on lock server 2024-03-26T20:39:11.720 DEBUG:teuthology.orchestra.run.smithi174:> uname -m 2024-03-26T20:39:11.738 INFO:teuthology.orchestra.run.smithi174.stdout:x86_64 2024-03-26T20:39:11.738 DEBUG:teuthology.orchestra.run.smithi174:> cat /etc/os-release 2024-03-26T20:39:11.795 INFO:teuthology.orchestra.run.smithi174.stdout:NAME="CentOS Stream" 2024-03-26T20:39:11.795 INFO:teuthology.orchestra.run.smithi174.stdout:VERSION="9" 2024-03-26T20:39:11.795 INFO:teuthology.orchestra.run.smithi174.stdout:ID="centos" 2024-03-26T20:39:11.795 INFO:teuthology.orchestra.run.smithi174.stdout:ID_LIKE="rhel fedora" 2024-03-26T20:39:11.795 INFO:teuthology.orchestra.run.smithi174.stdout:VERSION_ID="9" 2024-03-26T20:39:11.795 INFO:teuthology.orchestra.run.smithi174.stdout:PLATFORM_ID="platform:el9" 2024-03-26T20:39:11.795 INFO:teuthology.orchestra.run.smithi174.stdout:PRETTY_NAME="CentOS Stream 9" 2024-03-26T20:39:11.795 INFO:teuthology.orchestra.run.smithi174.stdout:ANSI_COLOR="0;31" 2024-03-26T20:39:11.795 INFO:teuthology.orchestra.run.smithi174.stdout:LOGO="fedora-logo-icon" 2024-03-26T20:39:11.796 INFO:teuthology.orchestra.run.smithi174.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-03-26T20:39:11.796 INFO:teuthology.orchestra.run.smithi174.stdout:HOME_URL="https://centos.org/" 2024-03-26T20:39:11.796 INFO:teuthology.orchestra.run.smithi174.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-03-26T20:39:11.796 INFO:teuthology.orchestra.run.smithi174.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-03-26T20:39:11.796 INFO:teuthology.orchestra.run.smithi174.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-03-26T20:39:11.796 INFO:teuthology.lock.ops:Updating smithi174.front.sepia.ceph.com on lock server 2024-03-26T20:39:11.820 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-03-26T20:39:11.829 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-03-26T20:39:11.835 INFO:teuthology.task.internal:Checking for old test directory... 2024-03-26T20:39:11.836 DEBUG:teuthology.orchestra.run.smithi063:> test '!' -e /home/ubuntu/cephtest 2024-03-26T20:39:11.839 DEBUG:teuthology.orchestra.run.smithi174:> test '!' -e /home/ubuntu/cephtest 2024-03-26T20:39:11.854 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-03-26T20:39:11.860 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-03-26T20:39:11.860 DEBUG:teuthology.orchestra.run.smithi063:> test -z $(ls -A /var/lib/ceph) 2024-03-26T20:39:11.896 DEBUG:teuthology.orchestra.run.smithi174:> test -z $(ls -A /var/lib/ceph) 2024-03-26T20:39:11.983 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-03-26T20:39:12.039 INFO:teuthology.run_tasks:Running task kernel... 2024-03-26T20:39:12.057 INFO:teuthology.task.kernel:normalize config orig: {'kdb': True, 'sha1': 'distro'} 2024-03-26T20:39:12.057 INFO:teuthology.task.kernel:config {'host.a': {'kdb': True, 'sha1': 'distro'}, 'host.b': {'kdb': True, 'sha1': 'distro'}}, timeout 300 2024-03-26T20:39:12.058 DEBUG:teuthology.orchestra.run.smithi063:> test -f /run/.containerenv -o -f /.dockerenv 2024-03-26T20:39:12.058 DEBUG:teuthology.orchestra.run.smithi174:> test -f /run/.containerenv -o -f /.dockerenv 2024-03-26T20:39:12.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-26T20:39:12.074 DEBUG:teuthology.orchestra.run.smithi174:> uname -r 2024-03-26T20:39:12.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-26T20:39:12.076 DEBUG:teuthology.orchestra.run.smithi063:> uname -r 2024-03-26T20:39:12.131 INFO:teuthology.orchestra.run.smithi063.stdout:5.14.0-361.el9.x86_64 2024-03-26T20:39:12.132 INFO:teuthology.task.kernel:Running kernel on smithi063: 5.14.0-361.el9.x86_64 2024-03-26T20:39:12.132 DEBUG:teuthology.orchestra.run.smithi063:> sudo yum install -y kernel 2024-03-26T20:39:12.133 INFO:teuthology.orchestra.run.smithi174.stdout:5.14.0-361.el9.x86_64 2024-03-26T20:39:12.133 INFO:teuthology.task.kernel:Running kernel on smithi174: 5.14.0-361.el9.x86_64 2024-03-26T20:39:12.133 DEBUG:teuthology.orchestra.run.smithi174:> sudo yum install -y kernel 2024-03-26T20:39:14.407 INFO:teuthology.orchestra.run.smithi174.stdout:CentOS-9 - AppStream 11 MB/s | 6.7 MB 00:00 2024-03-26T20:39:14.447 INFO:teuthology.orchestra.run.smithi063.stdout:CentOS-9 - AppStream 11 MB/s | 6.7 MB 00:00 2024-03-26T20:39:15.872 INFO:teuthology.orchestra.run.smithi063.stdout:CentOS-9 - BaseOS 6.5 MB/s | 2.0 MB 00:00 2024-03-26T20:39:15.899 INFO:teuthology.orchestra.run.smithi174.stdout:CentOS-9 - BaseOS 5.1 MB/s | 2.0 MB 00:00 2024-03-26T20:39:16.723 INFO:teuthology.orchestra.run.smithi174.stdout:CentOS Stream 9 - BaseOS 16 MB/s | 8.0 MB 00:00 2024-03-26T20:39:16.779 INFO:teuthology.orchestra.run.smithi063.stdout:CentOS Stream 9 - BaseOS 13 MB/s | 8.0 MB 00:00 2024-03-26T20:39:20.433 INFO:teuthology.orchestra.run.smithi174.stdout:CentOS Stream 9 - AppStream 7.1 MB/s | 19 MB 00:02 2024-03-26T20:39:20.464 INFO:teuthology.orchestra.run.smithi063.stdout:CentOS Stream 9 - AppStream 7.3 MB/s | 19 MB 00:02 2024-03-26T20:39:24.858 INFO:teuthology.orchestra.run.smithi174.stdout:CentOS Stream 9 - CRB 6.2 MB/s | 6.1 MB 00:00 2024-03-26T20:39:27.058 INFO:teuthology.orchestra.run.smithi174.stdout:CentOS Stream 9 - Extras packages 15 kB/s | 16 kB 00:01 2024-03-26T20:39:29.012 INFO:teuthology.orchestra.run.smithi174.stdout:Extra Packages for Enterprise Linux 11 MB/s | 21 MB 00:01 2024-03-26T20:39:32.248 INFO:teuthology.orchestra.run.smithi063.stdout:CentOS Stream 9 - CRB 776 kB/s | 6.1 MB 00:08 2024-03-26T20:39:33.639 INFO:teuthology.orchestra.run.smithi063.stdout:CentOS Stream 9 - Extras packages 82 kB/s | 16 kB 00:00 2024-03-26T20:39:33.934 INFO:teuthology.orchestra.run.smithi174.stdout:lab-extras 14 kB/s | 1.7 kB 00:00 2024-03-26T20:39:34.982 INFO:teuthology.orchestra.run.smithi063.stdout:Extra Packages for Enterprise Linux 17 MB/s | 21 MB 00:01 2024-03-26T20:39:36.191 INFO:teuthology.orchestra.run.smithi174.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-03-26T20:39:36.191 INFO:teuthology.orchestra.run.smithi174.stdout:Package kernel-5.14.0-361.el9.x86_64 is already installed. 2024-03-26T20:39:36.341 INFO:teuthology.orchestra.run.smithi174.stdout:Dependencies resolved. 2024-03-26T20:39:36.346 INFO:teuthology.orchestra.run.smithi174.stdout:================================================================================ 2024-03-26T20:39:36.347 INFO:teuthology.orchestra.run.smithi174.stdout: Package Architecture Version Repository Size 2024-03-26T20:39:36.347 INFO:teuthology.orchestra.run.smithi174.stdout:================================================================================ 2024-03-26T20:39:36.347 INFO:teuthology.orchestra.run.smithi174.stdout:Installing: 2024-03-26T20:39:36.347 INFO:teuthology.orchestra.run.smithi174.stdout: kernel x86_64 5.14.0-432.el9 baseos 66 k 2024-03-26T20:39:36.347 INFO:teuthology.orchestra.run.smithi174.stdout:Installing dependencies: 2024-03-26T20:39:36.347 INFO:teuthology.orchestra.run.smithi174.stdout: kernel-core x86_64 5.14.0-432.el9 baseos 15 M 2024-03-26T20:39:36.347 INFO:teuthology.orchestra.run.smithi174.stdout: kernel-modules x86_64 5.14.0-432.el9 baseos 33 M 2024-03-26T20:39:36.347 INFO:teuthology.orchestra.run.smithi174.stdout: kernel-modules-core x86_64 5.14.0-432.el9 baseos 27 M 2024-03-26T20:39:36.347 INFO:teuthology.orchestra.run.smithi174.stdout: 2024-03-26T20:39:36.347 INFO:teuthology.orchestra.run.smithi174.stdout:Transaction Summary 2024-03-26T20:39:36.347 INFO:teuthology.orchestra.run.smithi174.stdout:================================================================================ 2024-03-26T20:39:36.347 INFO:teuthology.orchestra.run.smithi174.stdout:Install 4 Packages 2024-03-26T20:39:36.347 INFO:teuthology.orchestra.run.smithi174.stdout: 2024-03-26T20:39:36.348 INFO:teuthology.orchestra.run.smithi174.stdout:Total download size: 76 M 2024-03-26T20:39:36.348 INFO:teuthology.orchestra.run.smithi174.stdout:Installed size: 123 M 2024-03-26T20:39:36.348 INFO:teuthology.orchestra.run.smithi174.stdout:Downloading Packages: 2024-03-26T20:39:36.535 INFO:teuthology.orchestra.run.smithi174.stdout:(1/4): kernel-5.14.0-432.el9.x86_64.rpm 520 kB/s | 66 kB 00:00 2024-03-26T20:39:39.585 INFO:teuthology.orchestra.run.smithi174.stdout:(2/4): kernel-core-5.14.0-432.el9.x86_64.rpm 4.7 MB/s | 15 MB 00:03 2024-03-26T20:39:40.103 INFO:teuthology.orchestra.run.smithi063.stdout:lab-extras 46 kB/s | 1.7 kB 00:00 2024-03-26T20:39:41.085 INFO:teuthology.orchestra.run.smithi174.stdout:(3/4): kernel-modules-core-5.14.0-432.el9.x86_6 6.0 MB/s | 27 MB 00:04 2024-03-26T20:39:41.818 INFO:teuthology.orchestra.run.smithi174.stdout:(4/4): kernel-modules-5.14.0-432.el9.x86_64.rpm 6.2 MB/s | 33 MB 00:05 2024-03-26T20:39:41.819 INFO:teuthology.orchestra.run.smithi174.stdout:-------------------------------------------------------------------------------- 2024-03-26T20:39:41.819 INFO:teuthology.orchestra.run.smithi174.stdout:Total 14 MB/s | 76 MB 00:05 2024-03-26T20:39:42.315 INFO:teuthology.orchestra.run.smithi063.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-03-26T20:39:42.315 INFO:teuthology.orchestra.run.smithi063.stdout:Package kernel-5.14.0-361.el9.x86_64 is already installed. 2024-03-26T20:39:42.420 INFO:teuthology.orchestra.run.smithi063.stdout:Dependencies resolved. 2024-03-26T20:39:42.430 INFO:teuthology.orchestra.run.smithi063.stdout:================================================================================ 2024-03-26T20:39:42.430 INFO:teuthology.orchestra.run.smithi063.stdout: Package Architecture Version Repository Size 2024-03-26T20:39:42.430 INFO:teuthology.orchestra.run.smithi063.stdout:================================================================================ 2024-03-26T20:39:42.430 INFO:teuthology.orchestra.run.smithi063.stdout:Installing: 2024-03-26T20:39:42.430 INFO:teuthology.orchestra.run.smithi063.stdout: kernel x86_64 5.14.0-432.el9 baseos 66 k 2024-03-26T20:39:42.430 INFO:teuthology.orchestra.run.smithi063.stdout:Installing dependencies: 2024-03-26T20:39:42.430 INFO:teuthology.orchestra.run.smithi063.stdout: kernel-core x86_64 5.14.0-432.el9 baseos 15 M 2024-03-26T20:39:42.431 INFO:teuthology.orchestra.run.smithi063.stdout: kernel-modules x86_64 5.14.0-432.el9 baseos 33 M 2024-03-26T20:39:42.431 INFO:teuthology.orchestra.run.smithi063.stdout: kernel-modules-core x86_64 5.14.0-432.el9 baseos 27 M 2024-03-26T20:39:42.431 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-03-26T20:39:42.431 INFO:teuthology.orchestra.run.smithi063.stdout:Transaction Summary 2024-03-26T20:39:42.431 INFO:teuthology.orchestra.run.smithi063.stdout:================================================================================ 2024-03-26T20:39:42.431 INFO:teuthology.orchestra.run.smithi063.stdout:Install 4 Packages 2024-03-26T20:39:42.431 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-03-26T20:39:42.431 INFO:teuthology.orchestra.run.smithi063.stdout:Total download size: 76 M 2024-03-26T20:39:42.432 INFO:teuthology.orchestra.run.smithi063.stdout:Installed size: 123 M 2024-03-26T20:39:42.432 INFO:teuthology.orchestra.run.smithi063.stdout:Downloading Packages: 2024-03-26T20:39:42.488 INFO:teuthology.orchestra.run.smithi174.stdout:Running transaction check 2024-03-26T20:39:42.567 INFO:teuthology.orchestra.run.smithi063.stdout:(1/4): kernel-5.14.0-432.el9.x86_64.rpm 667 kB/s | 66 kB 00:00 2024-03-26T20:39:42.861 INFO:teuthology.orchestra.run.smithi174.stdout:Transaction check succeeded. 2024-03-26T20:39:42.861 INFO:teuthology.orchestra.run.smithi174.stdout:Running transaction test 2024-03-26T20:39:42.901 INFO:teuthology.orchestra.run.smithi063.stdout:(2/4): kernel-core-5.14.0-432.el9.x86_64.rpm 34 MB/s | 15 MB 00:00 2024-03-26T20:39:43.318 INFO:teuthology.orchestra.run.smithi063.stdout:(3/4): kernel-modules-5.14.0-432.el9.x86_64.rpm 39 MB/s | 33 MB 00:00 2024-03-26T20:39:43.568 INFO:teuthology.orchestra.run.smithi063.stdout:(4/4): kernel-modules-core-5.14.0-432.el9.x86_6 27 MB/s | 27 MB 00:01 2024-03-26T20:39:43.568 INFO:teuthology.orchestra.run.smithi063.stdout:-------------------------------------------------------------------------------- 2024-03-26T20:39:43.568 INFO:teuthology.orchestra.run.smithi063.stdout:Total 67 MB/s | 76 MB 00:01 2024-03-26T20:39:43.868 INFO:teuthology.orchestra.run.smithi174.stdout:Transaction test succeeded. 2024-03-26T20:39:43.868 INFO:teuthology.orchestra.run.smithi174.stdout:Running transaction 2024-03-26T20:39:44.262 INFO:teuthology.orchestra.run.smithi063.stdout:Running transaction check 2024-03-26T20:39:44.694 INFO:teuthology.orchestra.run.smithi063.stdout:Transaction check succeeded. 2024-03-26T20:39:44.694 INFO:teuthology.orchestra.run.smithi063.stdout:Running transaction test 2024-03-26T20:39:45.724 INFO:teuthology.orchestra.run.smithi063.stdout:Transaction test succeeded. 2024-03-26T20:39:45.724 INFO:teuthology.orchestra.run.smithi063.stdout:Running transaction 2024-03-26T20:39:46.331 INFO:teuthology.orchestra.run.smithi174.stdout: Preparing : 1/1 2024-03-26T20:39:47.057 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : kernel-modules-core-5.14.0-432.el9.x86_64 1/4 2024-03-26T20:39:47.143 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : kernel-core-5.14.0-432.el9.x86_64 2/4 2024-03-26T20:39:48.331 INFO:teuthology.orchestra.run.smithi063.stdout: Preparing : 1/1 2024-03-26T20:39:48.958 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : kernel-modules-core-5.14.0-432.el9.x86_64 1/4 2024-03-26T20:39:49.051 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : kernel-core-5.14.0-432.el9.x86_64 2/4 2024-03-26T20:39:49.872 INFO:teuthology.orchestra.run.smithi174.stdout: Running scriptlet: kernel-core-5.14.0-432.el9.x86_64 2/4 2024-03-26T20:39:50.004 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : kernel-modules-5.14.0-432.el9.x86_64 3/4 2024-03-26T20:39:51.950 INFO:teuthology.orchestra.run.smithi063.stdout: Running scriptlet: kernel-core-5.14.0-432.el9.x86_64 2/4 2024-03-26T20:39:52.086 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : kernel-modules-5.14.0-432.el9.x86_64 3/4 2024-03-26T20:39:54.716 INFO:teuthology.orchestra.run.smithi174.stdout: Running scriptlet: kernel-modules-5.14.0-432.el9.x86_64 3/4 2024-03-26T20:39:54.821 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : kernel-5.14.0-432.el9.x86_64 4/4 2024-03-26T20:39:57.155 INFO:teuthology.orchestra.run.smithi063.stdout: Running scriptlet: kernel-modules-5.14.0-432.el9.x86_64 3/4 2024-03-26T20:39:57.195 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : kernel-5.14.0-432.el9.x86_64 4/4 2024-03-26T20:39:59.546 INFO:teuthology.orchestra.run.smithi174.stdout: Running scriptlet: kernel-modules-core-5.14.0-432.el9.x86_64 4/4 2024-03-26T20:40:02.289 INFO:teuthology.orchestra.run.smithi063.stdout: Running scriptlet: kernel-modules-core-5.14.0-432.el9.x86_64 4/4 2024-03-26T20:40:28.639 INFO:teuthology.orchestra.run.smithi174.stdout: Running scriptlet: kernel-core-5.14.0-432.el9.x86_64 4/4 2024-03-26T20:40:28.640 INFO:teuthology.orchestra.run.smithi174.stdout:dracut: Disabling early microcode, because kernel does not support it. CONFIG_MICROCODE_[AMD|INTEL]!=y 2024-03-26T20:40:28.640 INFO:teuthology.orchestra.run.smithi174.stdout: 2024-03-26T20:40:28.683 INFO:teuthology.orchestra.run.smithi174.stdout: Running scriptlet: kernel-modules-5.14.0-432.el9.x86_64 4/4 2024-03-26T20:40:29.089 INFO:teuthology.orchestra.run.smithi174.stdout: Running scriptlet: kernel-5.14.0-432.el9.x86_64 4/4 2024-03-26T20:40:29.089 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : kernel-5.14.0-432.el9.x86_64 1/4 2024-03-26T20:40:29.089 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : kernel-core-5.14.0-432.el9.x86_64 2/4 2024-03-26T20:40:29.089 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : kernel-modules-5.14.0-432.el9.x86_64 3/4 2024-03-26T20:40:29.495 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : kernel-modules-core-5.14.0-432.el9.x86_64 4/4 2024-03-26T20:40:29.496 INFO:teuthology.orchestra.run.smithi174.stdout: 2024-03-26T20:40:29.496 INFO:teuthology.orchestra.run.smithi174.stdout:Installed: 2024-03-26T20:40:29.496 INFO:teuthology.orchestra.run.smithi174.stdout: kernel-5.14.0-432.el9.x86_64 2024-03-26T20:40:29.496 INFO:teuthology.orchestra.run.smithi174.stdout: kernel-core-5.14.0-432.el9.x86_64 2024-03-26T20:40:29.496 INFO:teuthology.orchestra.run.smithi174.stdout: kernel-modules-5.14.0-432.el9.x86_64 2024-03-26T20:40:29.496 INFO:teuthology.orchestra.run.smithi174.stdout: kernel-modules-core-5.14.0-432.el9.x86_64 2024-03-26T20:40:29.496 INFO:teuthology.orchestra.run.smithi174.stdout: 2024-03-26T20:40:29.496 INFO:teuthology.orchestra.run.smithi174.stdout:Complete! 2024-03-26T20:40:29.793 DEBUG:teuthology.orchestra.run.smithi174:> rpm -q kernel | sort -rV | head -n 1 2024-03-26T20:40:29.930 INFO:teuthology.orchestra.run.smithi174.stdout:kernel-5.14.0-432.el9.x86_64 2024-03-26T20:40:29.930 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-432.el9.x86_64 2024-03-26T20:40:29.930 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-361.el9.x86_64 Expected: 5.14.0-432.el9.x86_64 2024-03-26T20:40:29.930 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-03-26T20:40:29.930 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-03-26T20:40:29.931 INFO:teuthology.task.kernel:Installing distro kernel on host.b... 2024-03-26T20:40:29.931 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi174.front.sepia.ceph.com, path=None, version=distro) 2024-03-26T20:40:29.931 DEBUG:teuthology.orchestra.run.smithi174:> rpm -q kernel | sort -rV | head -n 1 2024-03-26T20:40:30.010 INFO:teuthology.orchestra.run.smithi174.stdout:kernel-5.14.0-432.el9.x86_64 2024-03-26T20:40:30.011 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-432.el9.x86_64 2024-03-26T20:40:30.011 DEBUG:teuthology.orchestra.run.smithi174:> sudo rpm -qi grub2-tools 2024-03-26T20:40:30.052 INFO:teuthology.orchestra.run.smithi174.stdout:Name : grub2-tools 2024-03-26T20:40:30.053 INFO:teuthology.orchestra.run.smithi174.stdout:Epoch : 1 2024-03-26T20:40:30.053 INFO:teuthology.orchestra.run.smithi174.stdout:Version : 2.06 2024-03-26T20:40:30.053 INFO:teuthology.orchestra.run.smithi174.stdout:Release : 68.el9 2024-03-26T20:40:30.053 INFO:teuthology.orchestra.run.smithi174.stdout:Architecture: x86_64 2024-03-26T20:40:30.053 INFO:teuthology.orchestra.run.smithi174.stdout:Install Date: Fri 01 Sep 2023 09:14:30 PM UTC 2024-03-26T20:40:30.053 INFO:teuthology.orchestra.run.smithi174.stdout:Group : Unspecified 2024-03-26T20:40:30.053 INFO:teuthology.orchestra.run.smithi174.stdout:Size : 8264254 2024-03-26T20:40:30.053 INFO:teuthology.orchestra.run.smithi174.stdout:License : GPLv3+ 2024-03-26T20:40:30.053 INFO:teuthology.orchestra.run.smithi174.stdout:Signature : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d 2024-03-26T20:40:30.053 INFO:teuthology.orchestra.run.smithi174.stdout:Source RPM : grub2-2.06-68.el9.src.rpm 2024-03-26T20:40:30.053 INFO:teuthology.orchestra.run.smithi174.stdout:Build Date : Wed 02 Aug 2023 03:15:48 PM UTC 2024-03-26T20:40:30.053 INFO:teuthology.orchestra.run.smithi174.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-03-26T20:40:30.053 INFO:teuthology.orchestra.run.smithi174.stdout:Packager : builder@centos.org 2024-03-26T20:40:30.054 INFO:teuthology.orchestra.run.smithi174.stdout:Vendor : CentOS 2024-03-26T20:40:30.054 INFO:teuthology.orchestra.run.smithi174.stdout:URL : http://www.gnu.org/software/grub/ 2024-03-26T20:40:30.054 INFO:teuthology.orchestra.run.smithi174.stdout:Summary : Support tools for GRUB. 2024-03-26T20:40:30.054 INFO:teuthology.orchestra.run.smithi174.stdout:Description : 2024-03-26T20:40:30.054 INFO:teuthology.orchestra.run.smithi174.stdout: 2024-03-26T20:40:30.054 INFO:teuthology.orchestra.run.smithi174.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-03-26T20:40:30.054 INFO:teuthology.orchestra.run.smithi174.stdout:customizable bootloader with modular architecture. It supports a rich 2024-03-26T20:40:30.054 INFO:teuthology.orchestra.run.smithi174.stdout:variety of kernel formats, file systems, computer architectures and 2024-03-26T20:40:30.054 INFO:teuthology.orchestra.run.smithi174.stdout:hardware devices. 2024-03-26T20:40:30.054 INFO:teuthology.orchestra.run.smithi174.stdout: 2024-03-26T20:40:30.054 INFO:teuthology.orchestra.run.smithi174.stdout:This subpackage provides tools for support of all platforms. 2024-03-26T20:40:30.055 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-03-26T20:40:30.055 INFO:teuthology.task.kernel:Updating grub on smithi174 to boot 5.14.0-432.el9.x86_64 2024-03-26T20:40:30.055 DEBUG:teuthology.orchestra.run.smithi174:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-03-26T20:40:30.685 INFO:teuthology.orchestra.run.smithi174.stderr:Generating grub configuration file ... 2024-03-26T20:40:32.107 INFO:teuthology.orchestra.run.smithi174.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-03-26T20:40:32.127 INFO:teuthology.orchestra.run.smithi174.stderr:done 2024-03-26T20:40:32.130 DEBUG:teuthology.orchestra.run.smithi174:> mktemp 2024-03-26T20:40:32.145 INFO:teuthology.orchestra.run.smithi174.stdout:/tmp/tmp.kSo20k6xW1 2024-03-26T20:40:32.146 DEBUG:teuthology.orchestra.run.smithi174:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.kSo20k6xW1 2024-03-26T20:40:32.214 DEBUG:teuthology.orchestra.run.smithi174:> sudo chmod 0666 /tmp/tmp.kSo20k6xW1 2024-03-26T20:40:32.350 DEBUG:teuthology.orchestra.remote:smithi174:/tmp/tmp.kSo20k6xW1 is 6KB 2024-03-26T20:40:32.399 DEBUG:teuthology.orchestra.run.smithi174:> rm -fr /tmp/tmp.kSo20k6xW1 2024-03-26T20:40:32.413 DEBUG:teuthology.orchestra.run.smithi174:> sudo /bin/ls /boot/loader/entries || true 2024-03-26T20:40:32.482 INFO:teuthology.orchestra.run.smithi174.stdout:15949e3c88704300882482eddfd7baae-0-rescue.conf 2024-03-26T20:40:32.482 INFO:teuthology.orchestra.run.smithi174.stdout:15949e3c88704300882482eddfd7baae-5.14.0-239.el9.x86_64.conf 2024-03-26T20:40:32.482 INFO:teuthology.orchestra.run.smithi174.stdout:15949e3c88704300882482eddfd7baae-5.14.0-361.el9.x86_64.conf 2024-03-26T20:40:32.482 INFO:teuthology.orchestra.run.smithi174.stdout:15949e3c88704300882482eddfd7baae-5.14.0-432.el9.x86_64.conf 2024-03-26T20:40:32.483 DEBUG:teuthology.orchestra.run.smithi174:> sudo grub2-set-default 15949e3c88704300882482eddfd7baae-5.14.0-432.el9.x86_64 2024-03-26T20:40:32.625 DEBUG:teuthology.orchestra.run.smithi174:> sudo shutdown -r now 2024-03-26T20:40:33.498 INFO:teuthology.orchestra.run.smithi063.stdout: Running scriptlet: kernel-core-5.14.0-432.el9.x86_64 4/4 2024-03-26T20:40:33.498 INFO:teuthology.orchestra.run.smithi063.stdout:dracut: Disabling early microcode, because kernel does not support it. CONFIG_MICROCODE_[AMD|INTEL]!=y 2024-03-26T20:40:33.498 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-03-26T20:40:33.544 INFO:teuthology.orchestra.run.smithi063.stdout: Running scriptlet: kernel-modules-5.14.0-432.el9.x86_64 4/4 2024-03-26T20:40:33.862 INFO:teuthology.orchestra.run.smithi063.stdout: Running scriptlet: kernel-5.14.0-432.el9.x86_64 4/4 2024-03-26T20:40:33.863 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : kernel-5.14.0-432.el9.x86_64 1/4 2024-03-26T20:40:33.863 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : kernel-core-5.14.0-432.el9.x86_64 2/4 2024-03-26T20:40:33.863 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : kernel-modules-5.14.0-432.el9.x86_64 3/4 2024-03-26T20:40:34.246 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : kernel-modules-core-5.14.0-432.el9.x86_64 4/4 2024-03-26T20:40:34.247 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-03-26T20:40:34.247 INFO:teuthology.orchestra.run.smithi063.stdout:Installed: 2024-03-26T20:40:34.247 INFO:teuthology.orchestra.run.smithi063.stdout: kernel-5.14.0-432.el9.x86_64 2024-03-26T20:40:34.247 INFO:teuthology.orchestra.run.smithi063.stdout: kernel-core-5.14.0-432.el9.x86_64 2024-03-26T20:40:34.247 INFO:teuthology.orchestra.run.smithi063.stdout: kernel-modules-5.14.0-432.el9.x86_64 2024-03-26T20:40:34.247 INFO:teuthology.orchestra.run.smithi063.stdout: kernel-modules-core-5.14.0-432.el9.x86_64 2024-03-26T20:40:34.247 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-03-26T20:40:34.247 INFO:teuthology.orchestra.run.smithi063.stdout:Complete! 2024-03-26T20:40:34.582 DEBUG:teuthology.orchestra.run.smithi063:> rpm -q kernel | sort -rV | head -n 1 2024-03-26T20:40:34.688 INFO:teuthology.orchestra.run.smithi063.stdout:kernel-5.14.0-432.el9.x86_64 2024-03-26T20:40:34.688 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-432.el9.x86_64 2024-03-26T20:40:34.688 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-361.el9.x86_64 Expected: 5.14.0-432.el9.x86_64 2024-03-26T20:40:34.689 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-03-26T20:40:34.689 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-03-26T20:40:34.689 INFO:teuthology.task.kernel:Installing distro kernel on host.a... 2024-03-26T20:40:34.689 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi063.front.sepia.ceph.com, path=None, version=distro) 2024-03-26T20:40:34.689 DEBUG:teuthology.orchestra.run.smithi063:> rpm -q kernel | sort -rV | head -n 1 2024-03-26T20:40:34.771 INFO:teuthology.orchestra.run.smithi063.stdout:kernel-5.14.0-432.el9.x86_64 2024-03-26T20:40:34.771 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-432.el9.x86_64 2024-03-26T20:40:34.772 DEBUG:teuthology.orchestra.run.smithi063:> sudo rpm -qi grub2-tools 2024-03-26T20:40:34.809 INFO:teuthology.orchestra.run.smithi063.stdout:Name : grub2-tools 2024-03-26T20:40:34.809 INFO:teuthology.orchestra.run.smithi063.stdout:Epoch : 1 2024-03-26T20:40:34.809 INFO:teuthology.orchestra.run.smithi063.stdout:Version : 2.06 2024-03-26T20:40:34.809 INFO:teuthology.orchestra.run.smithi063.stdout:Release : 68.el9 2024-03-26T20:40:34.809 INFO:teuthology.orchestra.run.smithi063.stdout:Architecture: x86_64 2024-03-26T20:40:34.809 INFO:teuthology.orchestra.run.smithi063.stdout:Install Date: Fri 01 Sep 2023 09:14:30 PM UTC 2024-03-26T20:40:34.809 INFO:teuthology.orchestra.run.smithi063.stdout:Group : Unspecified 2024-03-26T20:40:34.809 INFO:teuthology.orchestra.run.smithi063.stdout:Size : 8264254 2024-03-26T20:40:34.809 INFO:teuthology.orchestra.run.smithi063.stdout:License : GPLv3+ 2024-03-26T20:40:34.810 INFO:teuthology.orchestra.run.smithi063.stdout:Signature : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d 2024-03-26T20:40:34.810 INFO:teuthology.orchestra.run.smithi063.stdout:Source RPM : grub2-2.06-68.el9.src.rpm 2024-03-26T20:40:34.810 INFO:teuthology.orchestra.run.smithi063.stdout:Build Date : Wed 02 Aug 2023 03:15:48 PM UTC 2024-03-26T20:40:34.810 INFO:teuthology.orchestra.run.smithi063.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-03-26T20:40:34.810 INFO:teuthology.orchestra.run.smithi063.stdout:Packager : builder@centos.org 2024-03-26T20:40:34.810 INFO:teuthology.orchestra.run.smithi063.stdout:Vendor : CentOS 2024-03-26T20:40:34.810 INFO:teuthology.orchestra.run.smithi063.stdout:URL : http://www.gnu.org/software/grub/ 2024-03-26T20:40:34.810 INFO:teuthology.orchestra.run.smithi063.stdout:Summary : Support tools for GRUB. 2024-03-26T20:40:34.810 INFO:teuthology.orchestra.run.smithi063.stdout:Description : 2024-03-26T20:40:34.810 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-03-26T20:40:34.810 INFO:teuthology.orchestra.run.smithi063.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-03-26T20:40:34.810 INFO:teuthology.orchestra.run.smithi063.stdout:customizable bootloader with modular architecture. It supports a rich 2024-03-26T20:40:34.810 INFO:teuthology.orchestra.run.smithi063.stdout:variety of kernel formats, file systems, computer architectures and 2024-03-26T20:40:34.810 INFO:teuthology.orchestra.run.smithi063.stdout:hardware devices. 2024-03-26T20:40:34.811 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-03-26T20:40:34.811 INFO:teuthology.orchestra.run.smithi063.stdout:This subpackage provides tools for support of all platforms. 2024-03-26T20:40:34.811 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-03-26T20:40:34.812 INFO:teuthology.task.kernel:Updating grub on smithi063 to boot 5.14.0-432.el9.x86_64 2024-03-26T20:40:34.812 DEBUG:teuthology.orchestra.run.smithi063:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-03-26T20:40:35.474 INFO:teuthology.orchestra.run.smithi063.stderr:Generating grub configuration file ... 2024-03-26T20:40:36.837 INFO:teuthology.orchestra.run.smithi063.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-03-26T20:40:36.869 INFO:teuthology.orchestra.run.smithi063.stderr:done 2024-03-26T20:40:36.872 DEBUG:teuthology.orchestra.run.smithi063:> mktemp 2024-03-26T20:40:36.887 INFO:teuthology.orchestra.run.smithi063.stdout:/tmp/tmp.Y74Kgu0U5t 2024-03-26T20:40:36.888 DEBUG:teuthology.orchestra.run.smithi063:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.Y74Kgu0U5t 2024-03-26T20:40:36.959 DEBUG:teuthology.orchestra.run.smithi063:> sudo chmod 0666 /tmp/tmp.Y74Kgu0U5t 2024-03-26T20:40:37.102 DEBUG:teuthology.orchestra.remote:smithi063:/tmp/tmp.Y74Kgu0U5t is 6KB 2024-03-26T20:40:37.151 DEBUG:teuthology.orchestra.run.smithi063:> rm -fr /tmp/tmp.Y74Kgu0U5t 2024-03-26T20:40:37.165 DEBUG:teuthology.orchestra.run.smithi063:> sudo /bin/ls /boot/loader/entries || true 2024-03-26T20:40:37.242 INFO:teuthology.orchestra.run.smithi063.stdout:15949e3c88704300882482eddfd7baae-0-rescue.conf 2024-03-26T20:40:37.242 INFO:teuthology.orchestra.run.smithi063.stdout:15949e3c88704300882482eddfd7baae-5.14.0-239.el9.x86_64.conf 2024-03-26T20:40:37.242 INFO:teuthology.orchestra.run.smithi063.stdout:15949e3c88704300882482eddfd7baae-5.14.0-361.el9.x86_64.conf 2024-03-26T20:40:37.242 INFO:teuthology.orchestra.run.smithi063.stdout:15949e3c88704300882482eddfd7baae-5.14.0-432.el9.x86_64.conf 2024-03-26T20:40:37.244 DEBUG:teuthology.orchestra.run.smithi063:> sudo grub2-set-default 15949e3c88704300882482eddfd7baae-5.14.0-432.el9.x86_64 2024-03-26T20:40:37.391 DEBUG:teuthology.orchestra.run.smithi063:> sudo shutdown -r now 2024-03-26T20:41:02.650 INFO:teuthology.task.kernel:Checking client host.b for new kernel version... 2024-03-26T20:41:02.651 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi174.front.sepia.ceph.com' 2024-03-26T20:41:02.652 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi174.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-26T20:41:07.398 INFO:teuthology.task.kernel:Checking client host.a for new kernel version... 2024-03-26T20:41:07.399 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi063.front.sepia.ceph.com' 2024-03-26T20:41:07.399 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi063.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-26T20:41:25.791 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.63 2024-03-26T20:41:34.800 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi063.front.sepia.ceph.com' 2024-03-26T20:41:34.801 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi063.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-26T20:41:37.858 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.63 2024-03-26T20:41:49.870 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi063.front.sepia.ceph.com' 2024-03-26T20:41:49.871 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi063.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-26T20:42:02.667 DEBUG:teuthology.orchestra.remote:timed out 2024-03-26T20:42:11.674 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi174.front.sepia.ceph.com' 2024-03-26T20:42:11.675 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi174.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-26T20:42:13.073 DEBUG:teuthology.orchestra.run.smithi174:> true 2024-03-26T20:42:13.606 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi174.front.sepia.ceph.com' 2024-03-26T20:42:13.606 INFO:teuthology.task.kernel:Checking kernel version of host.b, want "5.14.0-432.el9.x86_64"... 2024-03-26T20:42:13.607 DEBUG:teuthology.orchestra.run.smithi174:> uname -r 2024-03-26T20:42:13.621 INFO:teuthology.orchestra.run.smithi174.stdout:5.14.0-432.el9.x86_64 2024-03-26T20:42:13.621 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-432.el9.x86_64 vs 5.14.0-432.el9.x86_64 2024-03-26T20:42:13.621 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-03-26T20:42:13.621 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-03-26T20:42:14.622 INFO:teuthology.task.kernel:Enabling kdb on host.b... 2024-03-26T20:42:14.622 DEBUG:teuthology.orchestra.run.smithi174:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-03-26T20:42:14.722 INFO:teuthology.orchestra.run.smithi174.stdout:ttyS1 2024-03-26T20:42:14.754 DEBUG:teuthology.parallel:result is None 2024-03-26T20:42:49.873 DEBUG:teuthology.orchestra.remote:timed out 2024-03-26T20:43:04.875 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi063.front.sepia.ceph.com' 2024-03-26T20:43:04.876 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi063.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-26T20:43:05.221 DEBUG:teuthology.orchestra.run.smithi063:> true 2024-03-26T20:43:05.625 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi063.front.sepia.ceph.com' 2024-03-26T20:43:05.625 INFO:teuthology.task.kernel:Checking kernel version of host.a, want "5.14.0-432.el9.x86_64"... 2024-03-26T20:43:05.625 DEBUG:teuthology.orchestra.run.smithi063:> uname -r 2024-03-26T20:43:05.639 INFO:teuthology.orchestra.run.smithi063.stdout:5.14.0-432.el9.x86_64 2024-03-26T20:43:05.640 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-432.el9.x86_64 vs 5.14.0-432.el9.x86_64 2024-03-26T20:43:05.640 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-03-26T20:43:05.640 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-03-26T20:43:06.641 INFO:teuthology.task.kernel:Enabling kdb on host.a... 2024-03-26T20:43:06.642 DEBUG:teuthology.orchestra.run.smithi063:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-03-26T20:43:06.722 INFO:teuthology.orchestra.run.smithi063.stdout:ttyS1 2024-03-26T20:43:06.762 DEBUG:teuthology.parallel:result is None 2024-03-26T20:43:06.763 INFO:teuthology.run_tasks:Running task internal.base... 2024-03-26T20:43:06.797 INFO:teuthology.task.internal:Creating test directory... 2024-03-26T20:43:06.797 DEBUG:teuthology.orchestra.run.smithi063:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-03-26T20:43:06.799 DEBUG:teuthology.orchestra.run.smithi174:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-03-26T20:43:06.818 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-03-26T20:43:06.825 INFO:teuthology.run_tasks:Running task internal.archive... 2024-03-26T20:43:06.831 INFO:teuthology.task.internal:Creating archive directory... 2024-03-26T20:43:06.831 DEBUG:teuthology.orchestra.run.smithi063:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-03-26T20:43:06.857 DEBUG:teuthology.orchestra.run.smithi174:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-03-26T20:43:06.928 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-03-26T20:43:06.935 INFO:teuthology.task.internal:Enabling coredump saving... 2024-03-26T20:43:06.935 DEBUG:teuthology.orchestra.run.smithi063:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-03-26T20:43:06.937 DEBUG:teuthology.orchestra.run.smithi174:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-03-26T20:43:06.978 INFO:teuthology.orchestra.run.smithi174.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-03-26T20:43:06.980 INFO:teuthology.orchestra.run.smithi063.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-03-26T20:43:06.992 INFO:teuthology.orchestra.run.smithi174.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-03-26T20:43:06.994 INFO:teuthology.orchestra.run.smithi063.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-03-26T20:43:06.995 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-03-26T20:43:07.002 INFO:teuthology.task.internal:Configuring sudo... 2024-03-26T20:43:07.003 DEBUG:teuthology.orchestra.run.smithi063:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-03-26T20:43:07.005 DEBUG:teuthology.orchestra.run.smithi174:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-03-26T20:43:07.064 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-03-26T20:43:07.074 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-03-26T20:43:07.074 DEBUG:teuthology.orchestra.run.smithi063:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-03-26T20:43:07.076 DEBUG:teuthology.orchestra.run.smithi174:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-03-26T20:43:07.120 DEBUG:teuthology.orchestra.run.smithi063:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-03-26T20:43:07.171 DEBUG:teuthology.orchestra.run.smithi063:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-03-26T20:43:07.225 DEBUG:teuthology.orchestra.run.smithi063:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-03-26T20:43:07.307 DEBUG:teuthology.orchestra.run.smithi063:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-03-26T20:43:07.349 DEBUG:teuthology.orchestra.run.smithi063:> set -ex 2024-03-26T20:43:07.349 DEBUG:teuthology.orchestra.run.smithi063:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-03-26T20:43:07.416 DEBUG:teuthology.orchestra.run.smithi174:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-03-26T20:43:07.457 DEBUG:teuthology.orchestra.run.smithi174:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-03-26T20:43:07.522 DEBUG:teuthology.orchestra.run.smithi174:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-03-26T20:43:07.563 DEBUG:teuthology.orchestra.run.smithi174:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-03-26T20:43:07.610 DEBUG:teuthology.orchestra.run.smithi174:> set -ex 2024-03-26T20:43:07.610 DEBUG:teuthology.orchestra.run.smithi174:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-03-26T20:43:07.679 DEBUG:teuthology.orchestra.run.smithi063:> sudo service rsyslog restart 2024-03-26T20:43:07.681 DEBUG:teuthology.orchestra.run.smithi174:> sudo service rsyslog restart 2024-03-26T20:43:07.720 INFO:teuthology.orchestra.run.smithi063.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-03-26T20:43:07.770 INFO:teuthology.orchestra.run.smithi174.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-03-26T20:43:08.083 INFO:teuthology.run_tasks:Running task internal.timer... 2024-03-26T20:43:08.091 INFO:teuthology.task.internal:Starting timer... 2024-03-26T20:43:08.091 INFO:teuthology.run_tasks:Running task pcp... 2024-03-26T20:43:08.107 INFO:teuthology.run_tasks:Running task selinux... 2024-03-26T20:43:08.116 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0', 'scontext=system_u:system_r:getty_t:s0']} 2024-03-26T20:43:08.117 DEBUG:teuthology.orchestra.run.smithi063:> sudo service auditd rotate 2024-03-26T20:43:08.187 INFO:teuthology.orchestra.run.smithi063.stdout:Rotating logs: 2024-03-26T20:43:08.188 DEBUG:teuthology.orchestra.run.smithi174:> sudo service auditd rotate 2024-03-26T20:43:08.228 INFO:teuthology.orchestra.run.smithi174.stdout:Rotating logs: 2024-03-26T20:43:08.230 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-03-26T20:43:08.231 DEBUG:teuthology.orchestra.run.smithi063:> /usr/sbin/getenforce 2024-03-26T20:43:08.251 INFO:teuthology.orchestra.run.smithi063.stdout:Permissive 2024-03-26T20:43:08.251 DEBUG:teuthology.orchestra.run.smithi174:> /usr/sbin/getenforce 2024-03-26T20:43:08.289 INFO:teuthology.orchestra.run.smithi174.stdout:Permissive 2024-03-26T20:43:08.290 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi063.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi174.front.sepia.ceph.com': 'permissive'} 2024-03-26T20:43:08.290 DEBUG:teuthology.orchestra.run.smithi063:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-03-26T20:43:08.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-26T20:43:08.319 DEBUG:teuthology.orchestra.run.smithi174:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-03-26T20:43:08.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-26T20:43:08.356 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-03-26T20:43:08.356 DEBUG:teuthology.orchestra.run.smithi063:> sudo /usr/sbin/setenforce permissive 2024-03-26T20:43:08.395 DEBUG:teuthology.orchestra.run.smithi174:> sudo /usr/sbin/setenforce permissive 2024-03-26T20:43:08.433 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-03-26T20:43:08.443 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2024-03-26T20:43:08.448 INFO:teuthology.repo_utils:Fetching git.ceph.com_ceph-cm-ansible_main from origin 2024-03-26T20:43:08.576 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-03-26T20:43:08.592 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-03-26T20:43:08.594 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi063.front.sepia.ceph.com,smithi174.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-03-26T20:47:27.141 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi063.front.sepia.ceph.com'), Remote(name='ubuntu@smithi174.front.sepia.ceph.com')] 2024-03-26T20:47:27.143 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi063.front.sepia.ceph.com' 2024-03-26T20:47:27.144 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi063.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-26T20:47:27.236 DEBUG:teuthology.orchestra.run.smithi063:> true 2024-03-26T20:47:27.314 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi063.front.sepia.ceph.com' 2024-03-26T20:47:27.314 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi174.front.sepia.ceph.com' 2024-03-26T20:47:27.315 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi174.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-26T20:47:27.413 DEBUG:teuthology.orchestra.run.smithi174:> true 2024-03-26T20:47:27.488 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi174.front.sepia.ceph.com' 2024-03-26T20:47:27.489 INFO:teuthology.run_tasks:Running task clock... 2024-03-26T20:47:27.500 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-03-26T20:47:27.500 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-03-26T20:47:27.501 DEBUG:teuthology.orchestra.run.smithi063:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-03-26T20:47:27.503 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-03-26T20:47:27.504 DEBUG:teuthology.orchestra.run.smithi174:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-03-26T20:47:27.534 INFO:teuthology.orchestra.run.smithi063.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-03-26T20:47:27.552 INFO:teuthology.orchestra.run.smithi063.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-03-26T20:47:27.559 INFO:teuthology.orchestra.run.smithi174.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-03-26T20:47:27.575 INFO:teuthology.orchestra.run.smithi174.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-03-26T20:47:27.582 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: ntpd: command not found 2024-03-26T20:47:27.596 INFO:teuthology.orchestra.run.smithi063.stdout:506 Cannot talk to daemon 2024-03-26T20:47:27.604 INFO:teuthology.orchestra.run.smithi174.stderr:sudo: ntpd: command not found 2024-03-26T20:47:27.613 INFO:teuthology.orchestra.run.smithi063.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-03-26T20:47:27.618 INFO:teuthology.orchestra.run.smithi174.stdout:506 Cannot talk to daemon 2024-03-26T20:47:27.629 INFO:teuthology.orchestra.run.smithi063.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-03-26T20:47:27.634 INFO:teuthology.orchestra.run.smithi174.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-03-26T20:47:27.651 INFO:teuthology.orchestra.run.smithi174.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-03-26T20:47:27.681 INFO:teuthology.orchestra.run.smithi063.stderr:bash: line 1: ntpq: command not found 2024-03-26T20:47:27.686 INFO:teuthology.orchestra.run.smithi063.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-03-26T20:47:27.686 INFO:teuthology.orchestra.run.smithi063.stdout:=============================================================================== 2024-03-26T20:47:27.686 INFO:teuthology.orchestra.run.smithi063.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-26T20:47:27.686 INFO:teuthology.orchestra.run.smithi063.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-26T20:47:27.686 INFO:teuthology.orchestra.run.smithi063.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-26T20:47:27.687 INFO:teuthology.orchestra.run.smithi063.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-26T20:47:27.700 INFO:teuthology.orchestra.run.smithi174.stderr:bash: line 1: ntpq: command not found 2024-03-26T20:47:27.703 INFO:teuthology.orchestra.run.smithi174.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-03-26T20:47:27.704 INFO:teuthology.orchestra.run.smithi174.stdout:=============================================================================== 2024-03-26T20:47:27.704 INFO:teuthology.orchestra.run.smithi174.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-26T20:47:27.704 INFO:teuthology.orchestra.run.smithi174.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-26T20:47:27.704 INFO:teuthology.orchestra.run.smithi174.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-26T20:47:27.704 INFO:teuthology.orchestra.run.smithi174.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-26T20:47:27.704 INFO:teuthology.run_tasks:Running task cephadm... 2024-03-26T20:47:28.059 INFO:tasks.cephadm:Config: {'cephadm_branch': 'v17.2.0', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'image': 'quay.io/ceph/ceph:v17.2.0', 'roleless': True, 'conf': {'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': '42fc52b86b14d4f88c40d75eb7102e0ab3918f55'} 2024-03-26T20:47:28.059 INFO:tasks.cephadm:Cluster image is quay.io/ceph/ceph:v17.2.0 2024-03-26T20:47:28.060 INFO:tasks.cephadm:Cluster fsid is 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 2024-03-26T20:47:28.060 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-03-26T20:47:28.060 INFO:tasks.cephadm:No mon roles; fabricating mons 2024-03-26T20:47:28.060 INFO:tasks.cephadm:Monitor IPs: {'mon.smithi063': '172.21.15.63', 'mon.smithi174': '172.21.15.174'} 2024-03-26T20:47:28.060 INFO:tasks.cephadm:Normalizing hostnames... 2024-03-26T20:47:28.060 DEBUG:teuthology.orchestra.run.smithi063:> sudo hostname $(hostname -s) 2024-03-26T20:47:28.088 DEBUG:teuthology.orchestra.run.smithi174:> sudo hostname $(hostname -s) 2024-03-26T20:47:28.116 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref v17.2.0)... 2024-03-26T20:47:28.117 DEBUG:teuthology.orchestra.run.smithi063:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-03-26T20:47:28.451 INFO:teuthology.orchestra.run.smithi063.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Mar 26 20:47 /home/ubuntu/cephtest/cephadm 2024-03-26T20:47:28.451 DEBUG:teuthology.orchestra.run.smithi174:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-03-26T20:47:28.653 INFO:teuthology.orchestra.run.smithi174.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Mar 26 20:47 /home/ubuntu/cephtest/cephadm 2024-03-26T20:47:28.654 DEBUG:teuthology.orchestra.run.smithi063:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-03-26T20:47:28.670 DEBUG:teuthology.orchestra.run.smithi174:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-03-26T20:47:28.693 INFO:tasks.cephadm:Pulling image quay.io/ceph/ceph:v17.2.0 on all hosts... 2024-03-26T20:47:28.694 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-03-26T20:47:28.712 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-03-26T20:47:29.539 INFO:teuthology.orchestra.run.smithi063.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-03-26T20:47:29.600 INFO:teuthology.orchestra.run.smithi174.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-03-26T20:47:51.746 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-03-26T20:47:51.746 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-03-26T20:47:51.746 INFO:teuthology.orchestra.run.smithi063.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-03-26T20:47:51.746 INFO:teuthology.orchestra.run.smithi063.stdout: "repo_digests": [ 2024-03-26T20:47:51.747 INFO:teuthology.orchestra.run.smithi063.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-03-26T20:47:51.747 INFO:teuthology.orchestra.run.smithi063.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-03-26T20:47:51.747 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-03-26T20:47:51.747 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-03-26T20:47:58.326 INFO:teuthology.orchestra.run.smithi174.stdout:{ 2024-03-26T20:47:58.326 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-03-26T20:47:58.326 INFO:teuthology.orchestra.run.smithi174.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-03-26T20:47:58.326 INFO:teuthology.orchestra.run.smithi174.stdout: "repo_digests": [ 2024-03-26T20:47:58.326 INFO:teuthology.orchestra.run.smithi174.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-03-26T20:47:58.326 INFO:teuthology.orchestra.run.smithi174.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-03-26T20:47:58.326 INFO:teuthology.orchestra.run.smithi174.stdout: ] 2024-03-26T20:47:58.326 INFO:teuthology.orchestra.run.smithi174.stdout:} 2024-03-26T20:47:58.344 DEBUG:teuthology.orchestra.run.smithi063:> sudo mkdir -p /etc/ceph 2024-03-26T20:47:58.380 DEBUG:teuthology.orchestra.run.smithi174:> sudo mkdir -p /etc/ceph 2024-03-26T20:47:58.409 DEBUG:teuthology.orchestra.run.smithi063:> sudo chmod 777 /etc/ceph 2024-03-26T20:47:58.450 DEBUG:teuthology.orchestra.run.smithi174:> sudo chmod 777 /etc/ceph 2024-03-26T20:47:58.476 INFO:tasks.cephadm:Writing seed config... 2024-03-26T20:47:58.480 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-03-26T20:47:58.480 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-03-26T20:47:58.480 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-03-26T20:47:58.480 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-03-26T20:47:58.480 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-03-26T20:47:58.480 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-03-26T20:47:58.480 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-03-26T20:47:58.480 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-03-26T20:47:58.481 DEBUG:teuthology.orchestra.run.smithi063:> set -ex 2024-03-26T20:47:58.481 DEBUG:teuthology.orchestra.run.smithi063:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-03-26T20:47:58.506 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-03-26T20:47:58.507 DEBUG:teuthology.orchestra.run.smithi063:mon.smithi063> sudo journalctl -f -n 0 -u ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715@mon.smithi063.service 2024-03-26T20:47:58.549 INFO:tasks.cephadm:Bootstrapping... 2024-03-26T20:47:58.549 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 -v bootstrap --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-ip 172.21.15.63 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-03-26T20:47:58.751 INFO:teuthology.orchestra.run.smithi063.stderr:-------------------------------------------------------------------------------- 2024-03-26T20:47:58.751 INFO:teuthology.orchestra.run.smithi063.stderr:cephadm ['--image', 'quay.io/ceph/ceph:v17.2.0', '-v', 'bootstrap', '--fsid', '0ed4dd4c-ebb2-11ee-95d0-87774f69a715', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-ip', '172.21.15.63', '--skip-admin-label'] 2024-03-26T20:47:58.774 INFO:teuthology.orchestra.run.smithi063.stderr:/bin/podman: 4.9.4-dev 2024-03-26T20:47:58.776 INFO:teuthology.orchestra.run.smithi063.stderr:Verifying podman|docker is present... 2024-03-26T20:47:58.797 INFO:teuthology.orchestra.run.smithi063.stderr:/bin/podman: 4.9.4-dev 2024-03-26T20:47:58.799 INFO:teuthology.orchestra.run.smithi063.stderr:Verifying lvm2 is present... 2024-03-26T20:47:58.799 INFO:teuthology.orchestra.run.smithi063.stderr:Verifying time synchronization is in place... 2024-03-26T20:47:58.806 INFO:teuthology.orchestra.run.smithi063.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-03-26T20:47:58.811 INFO:teuthology.orchestra.run.smithi063.stderr:systemctl: inactive 2024-03-26T20:47:58.818 INFO:teuthology.orchestra.run.smithi063.stderr:systemctl: enabled 2024-03-26T20:47:58.824 INFO:teuthology.orchestra.run.smithi063.stderr:systemctl: active 2024-03-26T20:47:58.825 INFO:teuthology.orchestra.run.smithi063.stderr:Unit chronyd.service is enabled and running 2024-03-26T20:47:58.825 INFO:teuthology.orchestra.run.smithi063.stderr:Repeating the final host check... 2024-03-26T20:47:58.846 INFO:teuthology.orchestra.run.smithi063.stderr:/bin/podman: 4.9.4-dev 2024-03-26T20:47:58.848 INFO:teuthology.orchestra.run.smithi063.stderr:podman (/bin/podman) version 4.9.4 is present 2024-03-26T20:47:58.848 INFO:teuthology.orchestra.run.smithi063.stderr:systemctl is present 2024-03-26T20:47:58.848 INFO:teuthology.orchestra.run.smithi063.stderr:lvcreate is present 2024-03-26T20:47:58.853 INFO:teuthology.orchestra.run.smithi063.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-03-26T20:47:58.860 INFO:teuthology.orchestra.run.smithi063.stderr:systemctl: inactive 2024-03-26T20:47:58.866 INFO:teuthology.orchestra.run.smithi063.stderr:systemctl: enabled 2024-03-26T20:47:58.873 INFO:teuthology.orchestra.run.smithi063.stderr:systemctl: active 2024-03-26T20:47:58.873 INFO:teuthology.orchestra.run.smithi063.stderr:Unit chronyd.service is enabled and running 2024-03-26T20:47:58.873 INFO:teuthology.orchestra.run.smithi063.stderr:Host looks OK 2024-03-26T20:47:58.873 INFO:teuthology.orchestra.run.smithi063.stderr:Cluster fsid: 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 2024-03-26T20:47:58.873 INFO:teuthology.orchestra.run.smithi063.stderr:Acquiring lock 139830694899232 on /run/cephadm/0ed4dd4c-ebb2-11ee-95d0-87774f69a715.lock 2024-03-26T20:47:58.873 INFO:teuthology.orchestra.run.smithi063.stderr:Lock 139830694899232 acquired on /run/cephadm/0ed4dd4c-ebb2-11ee-95d0-87774f69a715.lock 2024-03-26T20:47:58.874 INFO:teuthology.orchestra.run.smithi063.stderr:Verifying IP 172.21.15.63 port 3300 ... 2024-03-26T20:47:58.874 INFO:teuthology.orchestra.run.smithi063.stderr:Verifying IP 172.21.15.63 port 6789 ... 2024-03-26T20:47:58.874 INFO:teuthology.orchestra.run.smithi063.stderr:Base mon IP is 172.21.15.63, final addrv is [v2:172.21.15.63:3300,v1:172.21.15.63:6789] 2024-03-26T20:47:58.876 INFO:teuthology.orchestra.run.smithi063.stderr:/sbin/ip: default via 172.21.15.254 dev enp3s0f1 proto dhcp src 172.21.15.63 metric 100 2024-03-26T20:47:58.876 INFO:teuthology.orchestra.run.smithi063.stderr:/sbin/ip: 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.63 metric 100 2024-03-26T20:47:58.878 INFO:teuthology.orchestra.run.smithi063.stderr:/sbin/ip: ::1 dev lo proto kernel metric 256 pref medium 2024-03-26T20:47:58.878 INFO:teuthology.orchestra.run.smithi063.stderr:/sbin/ip: fe80::/64 dev enp3s0f1 proto kernel metric 256 pref medium 2024-03-26T20:47:58.879 INFO:teuthology.orchestra.run.smithi063.stderr:/sbin/ip: default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 1024 expires 1792sec hoplimit 64 pref medium 2024-03-26T20:47:58.881 INFO:teuthology.orchestra.run.smithi063.stderr:/sbin/ip: 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-03-26T20:47:58.881 INFO:teuthology.orchestra.run.smithi063.stderr:/sbin/ip: inet6 ::1/128 scope host 2024-03-26T20:47:58.881 INFO:teuthology.orchestra.run.smithi063.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-03-26T20:47:58.881 INFO:teuthology.orchestra.run.smithi063.stderr:/sbin/ip: 5: enp3s0f1: mtu 1500 state UP qlen 1000 2024-03-26T20:47:58.881 INFO:teuthology.orchestra.run.smithi063.stderr:/sbin/ip: inet6 fe80::ec4:7aff:feb6:f8d7/64 scope link 2024-03-26T20:47:58.881 INFO:teuthology.orchestra.run.smithi063.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-03-26T20:47:58.882 INFO:teuthology.orchestra.run.smithi063.stderr:Mon IP `172.21.15.63` is in CIDR network `172.21.0.0/20` 2024-03-26T20:47:58.882 INFO:teuthology.orchestra.run.smithi063.stderr:- internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-03-26T20:47:58.882 INFO:teuthology.orchestra.run.smithi063.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-03-26T20:47:58.985 INFO:teuthology.orchestra.run.smithi063.stderr:/bin/podman: Trying to pull quay.io/ceph/ceph:v17.2.0... 2024-03-26T20:47:59.339 INFO:teuthology.orchestra.run.smithi063.stderr:/bin/podman: Getting image source signatures 2024-03-26T20:47:59.339 INFO:teuthology.orchestra.run.smithi063.stderr:/bin/podman: Copying blob sha256:33ca8fff7868c4dc0c11e09bca97c720eb9cfbab7221216754367dd8de70388a 2024-03-26T20:47:59.340 INFO:teuthology.orchestra.run.smithi063.stderr:/bin/podman: Copying blob sha256:89b4a75bc2d8500f15463747507c9623df43886c134463e7f0527e70900e7a7b 2024-03-26T20:47:59.340 INFO:teuthology.orchestra.run.smithi063.stderr:/bin/podman: Copying blob sha256:c32ab78b488d0c72f64eded765c0cf6b5bf2c75dab66cb62a9d367fa6ec42513 2024-03-26T20:47:59.340 INFO:teuthology.orchestra.run.smithi063.stderr:/bin/podman: Copying blob sha256:599d07cb321ff0a3c82224e1138fc685793fa69b93ed5780415751a5f7e4b8c2 2024-03-26T20:47:59.340 INFO:teuthology.orchestra.run.smithi063.stderr:/bin/podman: Copying blob sha256:a70843738bb77e1ab9c1f85969ebdfa55f178e746be081d1cb4f94011f69eb7c 2024-03-26T20:47:59.340 INFO:teuthology.orchestra.run.smithi063.stderr:/bin/podman: Copying config sha256:e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-03-26T20:47:59.474 INFO:teuthology.orchestra.run.smithi063.stderr:/bin/podman: Writing manifest to image destination 2024-03-26T20:47:59.593 INFO:teuthology.orchestra.run.smithi063.stderr:/bin/podman: e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-03-26T20:48:00.103 INFO:teuthology.orchestra.run.smithi063.stderr:ceph: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-03-26T20:48:00.717 INFO:teuthology.orchestra.run.smithi063.stderr:Ceph version: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-03-26T20:48:00.717 INFO:teuthology.orchestra.run.smithi063.stderr:Extracting ceph user uid/gid from container image... 2024-03-26T20:48:01.094 INFO:teuthology.orchestra.run.smithi063.stderr:stat: 167 167 2024-03-26T20:48:01.522 INFO:teuthology.orchestra.run.smithi063.stderr:Creating initial keys... 2024-03-26T20:48:01.838 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph-authtool: AQCBNANmB/nRLxAAvttStgWF9KptHf+C3vZzgQ== 2024-03-26T20:48:02.744 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph-authtool: AQCCNANmSskoJxAAcaiFbZ/W2uqGpya/QonWyg== 2024-03-26T20:48:03.723 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph-authtool: AQCDNANmODZwJhAACMdtBXFaoFeR3Q7rv7pPbQ== 2024-03-26T20:48:04.306 INFO:teuthology.orchestra.run.smithi063.stderr:Creating initial monmap... 2024-03-26T20:48:04.729 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: monmap file /tmp/monmap 2024-03-26T20:48:04.729 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/monmaptool: setting min_mon_release = octopus 2024-03-26T20:48:04.729 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: set fsid to 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 2024-03-26T20:48:04.729 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-03-26T20:48:05.218 INFO:teuthology.orchestra.run.smithi063.stderr:monmaptool for smithi063 [v2:172.21.15.63:3300,v1:172.21.15.63:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-03-26T20:48:05.218 INFO:teuthology.orchestra.run.smithi063.stderr:setting min_mon_release = octopus 2024-03-26T20:48:05.218 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/monmaptool: set fsid to 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 2024-03-26T20:48:05.218 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-03-26T20:48:05.218 INFO:teuthology.orchestra.run.smithi063.stderr: 2024-03-26T20:48:05.218 INFO:teuthology.orchestra.run.smithi063.stderr:Creating mon... 2024-03-26T20:48:12.325 INFO:teuthology.orchestra.run.smithi063.stderr:create mon.smithi063 on 2024-03-26T20:48:12.683 INFO:teuthology.orchestra.run.smithi063.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-03-26T20:48:12.954 INFO:teuthology.orchestra.run.smithi063.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715.target → /etc/systemd/system/ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715.target. 2024-03-26T20:48:12.955 INFO:teuthology.orchestra.run.smithi063.stderr:systemctl: Created symlink /etc/systemd/system/ceph.target.wants/ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715.target → /etc/systemd/system/ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715.target. 2024-03-26T20:48:13.473 INFO:teuthology.orchestra.run.smithi063.stderr:systemctl: Failed to reset failed state of unit ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715@mon.smithi063.service: Unit ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715@mon.smithi063.service not loaded. 2024-03-26T20:48:13.480 INFO:teuthology.orchestra.run.smithi063.stderr:systemctl: Created symlink /etc/systemd/system/ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715.target.wants/ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715@mon.smithi063.service → /etc/systemd/system/ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715@.service. 2024-03-26T20:48:14.490 INFO:teuthology.orchestra.run.smithi063.stderr:systemctl: disabled 2024-03-26T20:48:14.497 INFO:teuthology.orchestra.run.smithi063.stderr:systemctl: inactive 2024-03-26T20:48:14.497 INFO:teuthology.orchestra.run.smithi063.stderr:firewalld.service is not enabled 2024-03-26T20:48:14.497 INFO:teuthology.orchestra.run.smithi063.stderr:Not possible to enable service . firewalld.service is not available 2024-03-26T20:48:14.498 INFO:teuthology.orchestra.run.smithi063.stderr:Waiting for mon to start... 2024-03-26T20:48:14.498 INFO:teuthology.orchestra.run.smithi063.stderr:Waiting for mon... 2024-03-26T20:48:14.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:14 smithi063 systemd[1]: Started Ceph mon.smithi063 for 0ed4dd4c-ebb2-11ee-95d0-87774f69a715. 2024-03-26T20:48:14.986 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: cluster: 2024-03-26T20:48:14.987 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: id: 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 2024-03-26T20:48:14.987 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: health: HEALTH_OK 2024-03-26T20:48:14.987 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: 2024-03-26T20:48:14.987 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: services: 2024-03-26T20:48:14.987 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: mon: 1 daemons, quorum smithi063 (age 0.541387s) 2024-03-26T20:48:14.987 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: mgr: no daemons active 2024-03-26T20:48:14.988 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: osd: 0 osds: 0 up, 0 in 2024-03-26T20:48:14.988 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: 2024-03-26T20:48:14.988 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: data: 2024-03-26T20:48:14.988 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: pools: 0 pools, 0 pgs 2024-03-26T20:48:14.988 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: objects: 0 objects, 0 B 2024-03-26T20:48:14.989 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: usage: 0 B used, 0 B / 0 B avail 2024-03-26T20:48:14.989 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: pgs: 2024-03-26T20:48:14.989 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: 2024-03-26T20:48:15.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:15 smithi063 ceph-mon[24464]: mon.smithi063 is new leader, mons smithi063 in quorum (ranks 0) 2024-03-26T20:48:15.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:15 smithi063 ceph-mon[24464]: monmap e1: 1 mons at {smithi063=[v2:172.21.15.63:3300/0,v1:172.21.15.63:6789/0]} 2024-03-26T20:48:15.553 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:15 smithi063 ceph-mon[24464]: fsmap 2024-03-26T20:48:15.553 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:15 smithi063 ceph-mon[24464]: osdmap e1: 0 total, 0 up, 0 in 2024-03-26T20:48:15.553 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:15 smithi063 ceph-mon[24464]: mgrmap e1: no daemons active 2024-03-26T20:48:15.553 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:15 smithi063 ceph-mon[24464]: from='client.? 172.21.15.63:0/826967030' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-03-26T20:48:15.571 INFO:teuthology.orchestra.run.smithi063.stderr:mon is available 2024-03-26T20:48:15.572 INFO:teuthology.orchestra.run.smithi063.stderr:Assimilating anything we can from ceph.conf... 2024-03-26T20:48:16.100 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: 2024-03-26T20:48:16.100 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: [global] 2024-03-26T20:48:16.100 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: fsid = 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 2024-03-26T20:48:16.101 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: mon_host = [v2:172.21.15.63:3300,v1:172.21.15.63:6789] 2024-03-26T20:48:16.101 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-03-26T20:48:16.101 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-03-26T20:48:16.101 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-03-26T20:48:16.101 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-03-26T20:48:16.101 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: 2024-03-26T20:48:16.102 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: [mgr] 2024-03-26T20:48:16.102 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-03-26T20:48:16.102 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: 2024-03-26T20:48:16.102 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: [osd] 2024-03-26T20:48:16.102 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-03-26T20:48:16.103 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-03-26T20:48:16.634 INFO:teuthology.orchestra.run.smithi063.stderr:Generating new minimal ceph.conf... 2024-03-26T20:48:16.749 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:16 smithi063 ceph-mon[24464]: from='client.? 172.21.15.63:0/1102937146' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-03-26T20:48:16.750 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:16 smithi063 ceph-mon[24464]: from='client.? 172.21.15.63:0/1102937146' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-03-26T20:48:17.626 INFO:teuthology.orchestra.run.smithi063.stderr:Restarting the monitor... 2024-03-26T20:48:17.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:17 smithi063 ceph-mon[24464]: from='client.? 172.21.15.63:0/2352725241' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:48:17.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:17 smithi063 systemd[1]: Stopping Ceph mon.smithi063 for 0ed4dd4c-ebb2-11ee-95d0-87774f69a715... 2024-03-26T20:48:17.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:17 smithi063 bash[24695]: Error: no container with name or ID "ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-mon.smithi063" found: no such container 2024-03-26T20:48:18.510 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:18 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-mon-smithi063[24460]: 2024-03-26T20:48:18.098+0000 7fb106bcb700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi063 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-03-26T20:48:18.511 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:18 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-mon-smithi063[24460]: 2024-03-26T20:48:18.098+0000 7fb106bcb700 -1 mon.smithi063@0(leader) e1 *** Got Signal Terminated *** 2024-03-26T20:48:18.511 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:18 smithi063 podman[24708]: 2024-03-26 20:48:18.263332154 +0000 UTC m=+0.579317297 container died 39ce93c9c9721188e13198910250e6d0262ef70458700c21f0ac47be1216c134 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-mon-smithi063, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., distribution-scope=public, io.buildah.version=1.19.8, io.openshift.expose-services=, release=754, name=centos-stream, vcs-type=git, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.component=centos-stream-container, architecture=x86_64, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-03-26T20:48:18.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:18 smithi063 podman[24708]: 2024-03-26 20:48:18.623558087 +0000 UTC m=+0.939543231 container cleanup 39ce93c9c9721188e13198910250e6d0262ef70458700c21f0ac47be1216c134 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-mon-smithi063, ceph=True, version=8, release=754, name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, vendor=Red Hat, Inc., distribution-scope=public, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, io.openshift.expose-services=, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git) 2024-03-26T20:48:18.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:18 smithi063 bash[24708]: ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-mon-smithi063 2024-03-26T20:48:18.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:18 smithi063 bash[24736]: Error: no container with name or ID "ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-mon.smithi063" found: no such container 2024-03-26T20:48:20.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:20 smithi063 podman[24722]: 2024-03-26 20:48:20.117802157 +0000 UTC m=+1.848761566 container remove 39ce93c9c9721188e13198910250e6d0262ef70458700c21f0ac47be1216c134 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-mon-smithi063, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, distribution-scope=public, version=8, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., GIT_CLEAN=True, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, ceph=True, release=754, vcs-type=git, io.openshift.expose-services=, architecture=x86_64, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-03-26T20:48:20.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:20 smithi063 systemd[1]: ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715@mon.smithi063.service: Deactivated successfully. 2024-03-26T20:48:20.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:20 smithi063 systemd[1]: Stopped Ceph mon.smithi063 for 0ed4dd4c-ebb2-11ee-95d0-87774f69a715. 2024-03-26T20:48:20.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:20 smithi063 systemd[1]: Starting Ceph mon.smithi063 for 0ed4dd4c-ebb2-11ee-95d0-87774f69a715... 2024-03-26T20:48:20.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:20 smithi063 podman[24799]: 2024-03-26 20:48:20.259364851 +0000 UTC m=+0.019034293 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-03-26T20:48:21.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:21 smithi063 podman[24799]: 2024-03-26 20:48:21.283860456 +0000 UTC m=+1.043529880 container create e6bd58b6c0f97d6daffc88b86d5ed77e11a6cb4837d212e60cc534dd0b02c9a1 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-mon-smithi063, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, io.openshift.expose-services=, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, version=8, name=centos-stream, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, RELEASE=HEAD, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-03-26T20:48:22.303 INFO:teuthology.orchestra.run.smithi063.stderr:Setting mon public_network to 172.21.0.0/20 2024-03-26T20:48:22.428 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 podman[24799]: 2024-03-26 20:48:22.231165732 +0000 UTC m=+1.990835180 container init e6bd58b6c0f97d6daffc88b86d5ed77e11a6cb4837d212e60cc534dd0b02c9a1 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-mon-smithi063, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, distribution-scope=public, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-03-26T20:48:22.429 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 podman[24799]: 2024-03-26 20:48:22.234686736 +0000 UTC m=+1.994356162 container start e6bd58b6c0f97d6daffc88b86d5ed77e11a6cb4837d212e60cc534dd0b02c9a1 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-mon-smithi063, version=8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, com.redhat.component=centos-stream-container, ceph=True, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, release=754, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, name=centos-stream, architecture=x86_64) 2024-03-26T20:48:22.429 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: set uid:gid to 167:167 (ceph:ceph) 2024-03-26T20:48:22.429 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-03-26T20:48:22.429 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: pidfile_write: ignore empty --pid-file 2024-03-26T20:48:22.429 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: load: jerasure load: lrc 2024-03-26T20:48:22.429 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: RocksDB version: 6.15.5 2024-03-26T20:48:22.431 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-03-26T20:48:22.431 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Compile date Apr 18 2022 2024-03-26T20:48:22.431 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: DB SUMMARY 2024-03-26T20:48:22.431 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: DB Session ID: Q4GTWA61KMO6G73028E4 2024-03-26T20:48:22.431 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: CURRENT file: CURRENT 2024-03-26T20:48:22.431 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: IDENTITY file: IDENTITY 2024-03-26T20:48:22.431 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-03-26T20:48:22.431 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi063/store.db dir, Total Num: 1, files: 000008.sst 2024-03-26T20:48:22.432 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi063/store.db: 000010.log size: 82711 ; 2024-03-26T20:48:22.432 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.error_if_exists: 0 2024-03-26T20:48:22.432 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.create_if_missing: 0 2024-03-26T20:48:22.432 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.paranoid_checks: 1 2024-03-26T20:48:22.432 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-03-26T20:48:22.432 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.env: 0x55ad872c6860 2024-03-26T20:48:22.432 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.fs: Posix File System 2024-03-26T20:48:22.432 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.info_log: 0x55ad88373fa0 2024-03-26T20:48:22.432 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.max_file_opening_threads: 16 2024-03-26T20:48:22.433 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.statistics: (nil) 2024-03-26T20:48:22.433 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.use_fsync: 0 2024-03-26T20:48:22.433 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.max_log_file_size: 0 2024-03-26T20:48:22.433 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-03-26T20:48:22.433 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.log_file_time_to_roll: 0 2024-03-26T20:48:22.433 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.keep_log_file_num: 1000 2024-03-26T20:48:22.433 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.recycle_log_file_num: 0 2024-03-26T20:48:22.433 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.allow_fallocate: 1 2024-03-26T20:48:22.433 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.allow_mmap_reads: 0 2024-03-26T20:48:22.433 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.allow_mmap_writes: 0 2024-03-26T20:48:22.433 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.use_direct_reads: 0 2024-03-26T20:48:22.433 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-03-26T20:48:22.434 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.create_missing_column_families: 0 2024-03-26T20:48:22.434 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.db_log_dir: 2024-03-26T20:48:22.434 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-smithi063/store.db 2024-03-26T20:48:22.434 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.table_cache_numshardbits: 6 2024-03-26T20:48:22.434 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.WAL_ttl_seconds: 0 2024-03-26T20:48:22.434 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.WAL_size_limit_MB: 0 2024-03-26T20:48:22.434 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-03-26T20:48:22.434 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-03-26T20:48:22.434 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.is_fd_close_on_exec: 1 2024-03-26T20:48:22.434 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.advise_random_on_open: 1 2024-03-26T20:48:22.434 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.db_write_buffer_size: 0 2024-03-26T20:48:22.434 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.write_buffer_manager: 0x55ad88464330 2024-03-26T20:48:22.434 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-03-26T20:48:22.434 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-03-26T20:48:22.435 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-03-26T20:48:22.435 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.use_adaptive_mutex: 0 2024-03-26T20:48:22.435 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.rate_limiter: (nil) 2024-03-26T20:48:22.435 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-03-26T20:48:22.435 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.wal_recovery_mode: 2 2024-03-26T20:48:22.435 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.enable_thread_tracking: 0 2024-03-26T20:48:22.435 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.enable_pipelined_write: 0 2024-03-26T20:48:22.435 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.unordered_write: 0 2024-03-26T20:48:22.435 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-03-26T20:48:22.435 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-03-26T20:48:22.435 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-03-26T20:48:22.435 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-03-26T20:48:22.436 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.row_cache: None 2024-03-26T20:48:22.436 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.wal_filter: None 2024-03-26T20:48:22.436 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-03-26T20:48:22.436 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.allow_ingest_behind: 0 2024-03-26T20:48:22.436 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.preserve_deletes: 0 2024-03-26T20:48:22.436 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.two_write_queues: 0 2024-03-26T20:48:22.436 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.manual_wal_flush: 0 2024-03-26T20:48:22.436 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.atomic_flush: 0 2024-03-26T20:48:22.436 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-03-26T20:48:22.436 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.persist_stats_to_disk: 0 2024-03-26T20:48:22.436 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.write_dbid_to_manifest: 0 2024-03-26T20:48:22.436 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.log_readahead_size: 0 2024-03-26T20:48:22.436 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-03-26T20:48:22.437 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.best_efforts_recovery: 0 2024-03-26T20:48:22.437 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-03-26T20:48:22.437 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-03-26T20:48:22.437 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.allow_data_in_errors: 0 2024-03-26T20:48:22.437 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.db_host_id: __hostname__ 2024-03-26T20:48:22.437 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.max_background_jobs: 2 2024-03-26T20:48:22.437 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.max_background_compactions: -1 2024-03-26T20:48:22.437 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.max_subcompactions: 1 2024-03-26T20:48:22.437 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-03-26T20:48:22.437 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-03-26T20:48:22.437 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.delayed_write_rate : 16777216 2024-03-26T20:48:22.437 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.max_total_wal_size: 0 2024-03-26T20:48:22.437 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-03-26T20:48:22.438 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.stats_dump_period_sec: 600 2024-03-26T20:48:22.438 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.stats_persist_period_sec: 600 2024-03-26T20:48:22.438 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-03-26T20:48:22.438 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.max_open_files: -1 2024-03-26T20:48:22.438 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.bytes_per_sync: 0 2024-03-26T20:48:22.438 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.wal_bytes_per_sync: 0 2024-03-26T20:48:22.438 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.strict_bytes_per_sync: 0 2024-03-26T20:48:22.438 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.compaction_readahead_size: 0 2024-03-26T20:48:22.438 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.max_background_flushes: -1 2024-03-26T20:48:22.438 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Compression algorithms supported: 2024-03-26T20:48:22.438 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-03-26T20:48:22.438 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: kZSTD supported: 0 2024-03-26T20:48:22.439 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: kXpressCompression supported: 0 2024-03-26T20:48:22.439 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: kLZ4HCCompression supported: 1 2024-03-26T20:48:22.439 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: kLZ4Compression supported: 1 2024-03-26T20:48:22.439 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: kBZip2Compression supported: 0 2024-03-26T20:48:22.439 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: kZlibCompression supported: 1 2024-03-26T20:48:22.439 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: kSnappyCompression supported: 1 2024-03-26T20:48:22.439 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Fast CRC32 supported: Supported on x86 2024-03-26T20:48:22.439 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi063/store.db/MANIFEST-000009 2024-03-26T20:48:22.439 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-03-26T20:48:22.439 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-03-26T20:48:22.439 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.merge_operator: 2024-03-26T20:48:22.439 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.compaction_filter: None 2024-03-26T20:48:22.440 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.compaction_filter_factory: None 2024-03-26T20:48:22.440 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.sst_partitioner_factory: None 2024-03-26T20:48:22.440 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.memtable_factory: SkipListFactory 2024-03-26T20:48:22.440 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.table_factory: BlockBasedTable 2024-03-26T20:48:22.440 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55ad8833fd00) 2024-03-26T20:48:22.440 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: cache_index_and_filter_blocks: 1 2024-03-26T20:48:22.440 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-03-26T20:48:22.440 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-03-26T20:48:22.440 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: pin_top_level_index_and_filter: 1 2024-03-26T20:48:22.440 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: index_type: 0 2024-03-26T20:48:22.440 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: data_block_index_type: 0 2024-03-26T20:48:22.440 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: index_shortening: 1 2024-03-26T20:48:22.440 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: data_block_hash_table_util_ratio: 0.750000 2024-03-26T20:48:22.441 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: hash_index_allow_collision: 1 2024-03-26T20:48:22.441 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: checksum: 1 2024-03-26T20:48:22.441 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: no_block_cache: 0 2024-03-26T20:48:22.441 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: block_cache: 0x55ad883aa170 2024-03-26T20:48:22.441 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: block_cache_name: BinnedLRUCache 2024-03-26T20:48:22.441 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: block_cache_options: 2024-03-26T20:48:22.441 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: capacity : 536870912 2024-03-26T20:48:22.441 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: num_shard_bits : 4 2024-03-26T20:48:22.441 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: strict_capacity_limit : 0 2024-03-26T20:48:22.441 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: high_pri_pool_ratio: 0.000 2024-03-26T20:48:22.441 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: block_cache_compressed: (nil) 2024-03-26T20:48:22.441 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: persistent_cache: (nil) 2024-03-26T20:48:22.441 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: block_size: 4096 2024-03-26T20:48:22.442 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: block_size_deviation: 10 2024-03-26T20:48:22.442 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: block_restart_interval: 16 2024-03-26T20:48:22.442 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: index_block_restart_interval: 1 2024-03-26T20:48:22.442 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: metadata_block_size: 4096 2024-03-26T20:48:22.442 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: partition_filters: 0 2024-03-26T20:48:22.442 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: use_delta_encoding: 1 2024-03-26T20:48:22.442 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-03-26T20:48:22.442 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: whole_key_filtering: 1 2024-03-26T20:48:22.442 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: verify_compression: 0 2024-03-26T20:48:22.442 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: read_amp_bytes_per_bit: 0 2024-03-26T20:48:22.442 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: format_version: 4 2024-03-26T20:48:22.442 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: enable_index_compression: 1 2024-03-26T20:48:22.443 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: block_align: 0 2024-03-26T20:48:22.443 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.write_buffer_size: 33554432 2024-03-26T20:48:22.443 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.max_write_buffer_number: 2 2024-03-26T20:48:22.443 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.compression: NoCompression 2024-03-26T20:48:22.443 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.bottommost_compression: Disabled 2024-03-26T20:48:22.443 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.prefix_extractor: nullptr 2024-03-26T20:48:22.443 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-03-26T20:48:22.443 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.num_levels: 7 2024-03-26T20:48:22.443 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-03-26T20:48:22.443 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-03-26T20:48:22.444 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-03-26T20:48:22.444 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-03-26T20:48:22.444 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-03-26T20:48:22.444 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-03-26T20:48:22.444 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-03-26T20:48:22.444 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-03-26T20:48:22.444 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-03-26T20:48:22.444 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-03-26T20:48:22.444 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.compression_opts.window_bits: -14 2024-03-26T20:48:22.444 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.compression_opts.level: 32767 2024-03-26T20:48:22.444 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.compression_opts.strategy: 0 2024-03-26T20:48:22.444 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-03-26T20:48:22.444 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-03-26T20:48:22.444 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-03-26T20:48:22.444 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.compression_opts.enabled: false 2024-03-26T20:48:22.445 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-03-26T20:48:22.445 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-03-26T20:48:22.445 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-03-26T20:48:22.445 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.target_file_size_base: 67108864 2024-03-26T20:48:22.445 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.target_file_size_multiplier: 1 2024-03-26T20:48:22.445 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-03-26T20:48:22.445 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-03-26T20:48:22.445 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-03-26T20:48:22.445 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-03-26T20:48:22.445 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-03-26T20:48:22.445 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-03-26T20:48:22.445 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-03-26T20:48:22.445 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-03-26T20:48:22.445 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-03-26T20:48:22.446 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-03-26T20:48:22.446 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-03-26T20:48:22.446 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-03-26T20:48:22.446 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.arena_block_size: 4194304 2024-03-26T20:48:22.446 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-03-26T20:48:22.446 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-03-26T20:48:22.446 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-03-26T20:48:22.446 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.disable_auto_compactions: 0 2024-03-26T20:48:22.446 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-03-26T20:48:22.446 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-03-26T20:48:22.446 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-03-26T20:48:22.446 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-03-26T20:48:22.446 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-03-26T20:48:22.446 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-03-26T20:48:22.446 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-03-26T20:48:22.447 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-03-26T20:48:22.447 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-03-26T20:48:22.447 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-03-26T20:48:22.447 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.table_properties_collectors: 2024-03-26T20:48:22.447 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.inplace_update_support: 0 2024-03-26T20:48:22.447 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.inplace_update_num_locks: 10000 2024-03-26T20:48:22.447 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-03-26T20:48:22.447 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-03-26T20:48:22.447 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.memtable_huge_page_size: 0 2024-03-26T20:48:22.447 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.bloom_locality: 0 2024-03-26T20:48:22.447 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.max_successive_merges: 0 2024-03-26T20:48:22.447 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.optimize_filters_for_hits: 0 2024-03-26T20:48:22.448 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.paranoid_file_checks: 0 2024-03-26T20:48:22.448 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.force_consistency_checks: 1 2024-03-26T20:48:22.448 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.report_bg_io_stats: 0 2024-03-26T20:48:22.448 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.ttl: 2592000 2024-03-26T20:48:22.448 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.periodic_compaction_seconds: 0 2024-03-26T20:48:22.448 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.enable_blob_files: false 2024-03-26T20:48:22.448 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.min_blob_size: 0 2024-03-26T20:48:22.448 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.blob_file_size: 268435456 2024-03-26T20:48:22.448 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.blob_compression_type: NoCompression 2024-03-26T20:48:22.448 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.enable_blob_garbage_collection: false 2024-03-26T20:48:22.448 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-03-26T20:48:22.448 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi063/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-03-26T20:48:22.448 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 5 2024-03-26T20:48:22.448 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: [db/version_set.cc:4083] Creating manifest 13 2024-03-26T20:48:22.448 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: EVENT_LOG_v1 {"time_micros": 1711486102262385, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-03-26T20:48:22.449 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #10 mode 2 2024-03-26T20:48:22.449 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-03-26T20:48:22.449 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: EVENT_LOG_v1 {"time_micros": 1711486102263135, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 79393, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 77657, "index_size": 224, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 9884, "raw_average_key_size": 47, "raw_value_size": 71966, "raw_average_value_size": 344, "num_data_blocks": 10, "num_entries": 209, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1711486102, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "55255b81-e890-4c25-9d1d-44215cfc3b71", "db_session_id": "Q4GTWA61KMO6G73028E4"}} 2024-03-26T20:48:22.449 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: [db/version_set.cc:4083] Creating manifest 15 2024-03-26T20:48:22.449 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: EVENT_LOG_v1 {"time_micros": 1711486102263592, "job": 1, "event": "recovery_finished"} 2024-03-26T20:48:22.449 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-smithi063/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-03-26T20:48:22.449 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x55ad88390a80 2024-03-26T20:48:22.449 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: DB pointer 0x55ad883a0000 2024-03-26T20:48:22.449 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-03-26T20:48:22.449 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-03-26T20:48:22.449 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: ** DB Stats ** 2024-03-26T20:48:22.449 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-03-26T20:48:22.449 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-03-26T20:48:22.449 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-03-26T20:48:22.450 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-26T20:48:22.450 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-03-26T20:48:22.450 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-03-26T20:48:22.450 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-26T20:48:22.450 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: 2024-03-26T20:48:22.450 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: ** Compaction Stats [default] ** 2024-03-26T20:48:22.450 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-03-26T20:48:22.450 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-03-26T20:48:22.450 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: L0 2/0 79.33 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 147.0 0.00 0.00 1 0.001 0 0 2024-03-26T20:48:22.450 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: Sum 2/0 79.33 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 147.0 0.00 0.00 1 0.001 0 0 2024-03-26T20:48:22.450 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 147.0 0.00 0.00 1 0.001 0 0 2024-03-26T20:48:22.451 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: 2024-03-26T20:48:22.451 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: ** Compaction Stats [default] ** 2024-03-26T20:48:22.451 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-03-26T20:48:22.451 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-03-26T20:48:22.451 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 147.0 0.00 0.00 1 0.001 0 0 2024-03-26T20:48:22.451 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-03-26T20:48:22.451 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-03-26T20:48:22.451 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-03-26T20:48:22.451 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-03-26T20:48:22.451 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-03-26T20:48:22.451 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: AddFile(Keys): cumulative 0, interval 0 2024-03-26T20:48:22.451 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: Cumulative compaction: 0.00 GB write, 22.80 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-26T20:48:22.452 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: Interval compaction: 0.00 GB write, 22.80 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-26T20:48:22.452 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-03-26T20:48:22.452 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: 2024-03-26T20:48:22.452 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: ** File Read Latency Histogram By Level [default] ** 2024-03-26T20:48:22.452 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: 2024-03-26T20:48:22.452 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: ** Compaction Stats [default] ** 2024-03-26T20:48:22.452 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-03-26T20:48:22.452 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-03-26T20:48:22.452 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: L0 2/0 79.33 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 147.0 0.00 0.00 1 0.001 0 0 2024-03-26T20:48:22.452 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: Sum 2/0 79.33 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 147.0 0.00 0.00 1 0.001 0 0 2024-03-26T20:48:22.452 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-03-26T20:48:22.453 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: 2024-03-26T20:48:22.453 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: ** Compaction Stats [default] ** 2024-03-26T20:48:22.454 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-03-26T20:48:22.454 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-03-26T20:48:22.454 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 147.0 0.00 0.00 1 0.001 0 0 2024-03-26T20:48:22.454 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-03-26T20:48:22.454 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-03-26T20:48:22.454 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-03-26T20:48:22.454 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-03-26T20:48:22.454 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-03-26T20:48:22.454 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: AddFile(Keys): cumulative 0, interval 0 2024-03-26T20:48:22.454 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: Cumulative compaction: 0.00 GB write, 22.57 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-26T20:48:22.455 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-26T20:48:22.455 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-03-26T20:48:22.455 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: 2024-03-26T20:48:22.455 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: ** File Read Latency Histogram By Level [default] ** 2024-03-26T20:48:22.455 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: starting mon.smithi063 rank 0 at public addrs [v2:172.21.15.63:3300/0,v1:172.21.15.63:6789/0] at bind addrs [v2:172.21.15.63:3300/0,v1:172.21.15.63:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi063 fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 2024-03-26T20:48:22.455 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: mon.smithi063@-1(???) e1 preinit fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 2024-03-26T20:48:22.455 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: mon.smithi063@-1(???).mds e1 new map 2024-03-26T20:48:22.455 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: mon.smithi063@-1(???).mds e1 print_map 2024-03-26T20:48:22.455 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: e1 2024-03-26T20:48:22.456 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-03-26T20:48:22.456 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-03-26T20:48:22.456 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: legacy client fscid: -1 2024-03-26T20:48:22.456 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: 2024-03-26T20:48:22.456 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: No filesystems configured 2024-03-26T20:48:22.456 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: mon.smithi063@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-03-26T20:48:22.456 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: mon.smithi063@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-03-26T20:48:22.456 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: mon.smithi063@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-03-26T20:48:22.456 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: mon.smithi063@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-03-26T20:48:22.456 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: mon.smithi063@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-03-26T20:48:22.456 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 bash[24799]: e6bd58b6c0f97d6daffc88b86d5ed77e11a6cb4837d212e60cc534dd0b02c9a1 2024-03-26T20:48:22.456 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 systemd[1]: Started Ceph mon.smithi063 for 0ed4dd4c-ebb2-11ee-95d0-87774f69a715. 2024-03-26T20:48:22.457 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: mon.smithi063 is new leader, mons smithi063 in quorum (ranks 0) 2024-03-26T20:48:22.457 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: monmap e1: 1 mons at {smithi063=[v2:172.21.15.63:3300/0,v1:172.21.15.63:6789/0]} 2024-03-26T20:48:22.457 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: fsmap 2024-03-26T20:48:22.457 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: osdmap e1: 0 total, 0 up, 0 in 2024-03-26T20:48:22.457 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:22 smithi063 ceph-mon[24818]: mgrmap e1: no daemons active 2024-03-26T20:48:23.368 INFO:teuthology.orchestra.run.smithi063.stderr:Wrote config to /etc/ceph/ceph.conf 2024-03-26T20:48:23.368 INFO:teuthology.orchestra.run.smithi063.stderr:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-03-26T20:48:23.368 INFO:teuthology.orchestra.run.smithi063.stderr:Creating mgr... 2024-03-26T20:48:23.368 INFO:teuthology.orchestra.run.smithi063.stderr:Verifying port 9283 ... 2024-03-26T20:48:23.536 INFO:teuthology.orchestra.run.smithi063.stderr:systemctl: Failed to reset failed state of unit ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715@mgr.smithi063.lxhujt.service: Unit ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715@mgr.smithi063.lxhujt.service not loaded. 2024-03-26T20:48:23.542 INFO:teuthology.orchestra.run.smithi063.stderr:systemctl: Created symlink /etc/systemd/system/ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715.target.wants/ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715@mgr.smithi063.lxhujt.service → /etc/systemd/system/ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715@.service. 2024-03-26T20:48:24.150 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:23 smithi063 ceph-mon[24818]: from='client.? 172.21.15.63:0/1349591976' entity='client.admin' 2024-03-26T20:48:24.263 INFO:teuthology.orchestra.run.smithi063.stderr:systemctl: disabled 2024-03-26T20:48:24.270 INFO:teuthology.orchestra.run.smithi063.stderr:systemctl: inactive 2024-03-26T20:48:24.271 INFO:teuthology.orchestra.run.smithi063.stderr:firewalld.service is not enabled 2024-03-26T20:48:24.271 INFO:teuthology.orchestra.run.smithi063.stderr:Not possible to enable service . firewalld.service is not available 2024-03-26T20:48:24.278 INFO:teuthology.orchestra.run.smithi063.stderr:systemctl: disabled 2024-03-26T20:48:24.285 INFO:teuthology.orchestra.run.smithi063.stderr:systemctl: inactive 2024-03-26T20:48:24.286 INFO:teuthology.orchestra.run.smithi063.stderr:firewalld.service is not enabled 2024-03-26T20:48:24.286 INFO:teuthology.orchestra.run.smithi063.stderr:Not possible to open ports <[9283]>. firewalld.service is not available 2024-03-26T20:48:24.286 INFO:teuthology.orchestra.run.smithi063.stderr:Waiting for mgr to start... 2024-03-26T20:48:24.286 INFO:teuthology.orchestra.run.smithi063.stderr:Waiting for mgr... 2024-03-26T20:48:24.848 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: 2024-03-26T20:48:24.848 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: { 2024-03-26T20:48:24.848 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "fsid": "0ed4dd4c-ebb2-11ee-95d0-87774f69a715", 2024-03-26T20:48:24.849 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "health": { 2024-03-26T20:48:24.849 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-03-26T20:48:24.850 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "checks": {}, 2024-03-26T20:48:24.850 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "mutes": [] 2024-03-26T20:48:24.850 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: }, 2024-03-26T20:48:24.850 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-03-26T20:48:24.850 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "quorum": [ 2024-03-26T20:48:24.850 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: 0 2024-03-26T20:48:24.850 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: ], 2024-03-26T20:48:24.850 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "quorum_names": [ 2024-03-26T20:48:24.850 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "smithi063" 2024-03-26T20:48:24.850 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: ], 2024-03-26T20:48:24.850 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "quorum_age": 2, 2024-03-26T20:48:24.851 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "monmap": { 2024-03-26T20:48:24.851 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "epoch": 1, 2024-03-26T20:48:24.851 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-03-26T20:48:24.851 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "num_mons": 1 2024-03-26T20:48:24.851 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: }, 2024-03-26T20:48:24.851 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "osdmap": { 2024-03-26T20:48:24.851 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "epoch": 1, 2024-03-26T20:48:24.851 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "num_osds": 0, 2024-03-26T20:48:24.851 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-03-26T20:48:24.851 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-03-26T20:48:24.851 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-03-26T20:48:24.851 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-03-26T20:48:24.851 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-03-26T20:48:24.851 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: }, 2024-03-26T20:48:24.851 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "pgmap": { 2024-03-26T20:48:24.852 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-03-26T20:48:24.852 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-03-26T20:48:24.852 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "num_pools": 0, 2024-03-26T20:48:24.852 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "num_objects": 0, 2024-03-26T20:48:24.852 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-03-26T20:48:24.852 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-03-26T20:48:24.852 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-03-26T20:48:24.852 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "bytes_total": 0 2024-03-26T20:48:24.852 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: }, 2024-03-26T20:48:24.852 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "fsmap": { 2024-03-26T20:48:24.852 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "epoch": 1, 2024-03-26T20:48:24.852 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "by_rank": [], 2024-03-26T20:48:24.852 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "up:standby": 0 2024-03-26T20:48:24.852 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: }, 2024-03-26T20:48:24.853 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "mgrmap": { 2024-03-26T20:48:24.853 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "available": false, 2024-03-26T20:48:24.853 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-03-26T20:48:24.853 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "modules": [ 2024-03-26T20:48:24.853 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "iostat", 2024-03-26T20:48:24.853 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "nfs", 2024-03-26T20:48:24.853 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "restful" 2024-03-26T20:48:24.853 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: ], 2024-03-26T20:48:24.853 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "services": {} 2024-03-26T20:48:24.853 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: }, 2024-03-26T20:48:24.853 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "servicemap": { 2024-03-26T20:48:24.853 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "epoch": 1, 2024-03-26T20:48:24.853 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "modified": "2024-03-26T20:48:14.448691+0000", 2024-03-26T20:48:24.853 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "services": {} 2024-03-26T20:48:24.854 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: }, 2024-03-26T20:48:24.854 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "progress_events": {} 2024-03-26T20:48:24.854 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: } 2024-03-26T20:48:25.052 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:24 smithi063 ceph-mon[24818]: from='client.? 172.21.15.63:0/1512680959' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-03-26T20:48:25.375 INFO:teuthology.orchestra.run.smithi063.stderr:mgr not available, waiting (1/15)... 2024-03-26T20:48:27.890 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: 2024-03-26T20:48:27.890 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: { 2024-03-26T20:48:27.890 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "fsid": "0ed4dd4c-ebb2-11ee-95d0-87774f69a715", 2024-03-26T20:48:27.890 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "health": { 2024-03-26T20:48:27.890 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-03-26T20:48:27.890 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "checks": {}, 2024-03-26T20:48:27.890 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "mutes": [] 2024-03-26T20:48:27.890 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: }, 2024-03-26T20:48:27.890 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-03-26T20:48:27.890 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "quorum": [ 2024-03-26T20:48:27.891 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: 0 2024-03-26T20:48:27.891 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: ], 2024-03-26T20:48:27.891 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "quorum_names": [ 2024-03-26T20:48:27.891 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "smithi063" 2024-03-26T20:48:27.891 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: ], 2024-03-26T20:48:27.891 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "quorum_age": 5, 2024-03-26T20:48:27.891 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "monmap": { 2024-03-26T20:48:27.891 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "epoch": 1, 2024-03-26T20:48:27.891 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-03-26T20:48:27.891 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "num_mons": 1 2024-03-26T20:48:27.891 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: }, 2024-03-26T20:48:27.891 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "osdmap": { 2024-03-26T20:48:27.891 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "epoch": 1, 2024-03-26T20:48:27.891 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "num_osds": 0, 2024-03-26T20:48:27.891 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-03-26T20:48:27.892 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-03-26T20:48:27.892 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-03-26T20:48:27.892 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-03-26T20:48:27.892 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-03-26T20:48:27.892 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: }, 2024-03-26T20:48:27.892 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "pgmap": { 2024-03-26T20:48:27.892 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-03-26T20:48:27.892 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-03-26T20:48:27.892 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "num_pools": 0, 2024-03-26T20:48:27.892 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "num_objects": 0, 2024-03-26T20:48:27.892 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-03-26T20:48:27.892 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-03-26T20:48:27.892 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-03-26T20:48:27.892 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "bytes_total": 0 2024-03-26T20:48:27.892 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: }, 2024-03-26T20:48:27.893 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "fsmap": { 2024-03-26T20:48:27.893 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "epoch": 1, 2024-03-26T20:48:27.893 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "by_rank": [], 2024-03-26T20:48:27.893 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "up:standby": 0 2024-03-26T20:48:27.893 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: }, 2024-03-26T20:48:27.893 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "mgrmap": { 2024-03-26T20:48:27.893 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "available": false, 2024-03-26T20:48:27.893 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-03-26T20:48:27.893 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "modules": [ 2024-03-26T20:48:27.893 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "iostat", 2024-03-26T20:48:27.893 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "nfs", 2024-03-26T20:48:27.893 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "restful" 2024-03-26T20:48:27.893 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: ], 2024-03-26T20:48:27.894 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "services": {} 2024-03-26T20:48:27.894 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: }, 2024-03-26T20:48:27.894 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "servicemap": { 2024-03-26T20:48:27.894 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "epoch": 1, 2024-03-26T20:48:27.894 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "modified": "2024-03-26T20:48:14.448691+0000", 2024-03-26T20:48:27.894 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "services": {} 2024-03-26T20:48:27.894 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: }, 2024-03-26T20:48:27.894 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "progress_events": {} 2024-03-26T20:48:27.894 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: } 2024-03-26T20:48:28.446 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:28 smithi063 ceph-mon[24818]: from='client.? 172.21.15.63:0/346125418' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-03-26T20:48:28.448 INFO:teuthology.orchestra.run.smithi063.stderr:mgr not available, waiting (2/15)... 2024-03-26T20:48:30.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:30 smithi063 ceph-mon[24818]: Activating manager daemon smithi063.lxhujt 2024-03-26T20:48:30.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:30 smithi063 ceph-mon[24818]: mgrmap e2: smithi063.lxhujt(active, starting, since 0.00263496s) 2024-03-26T20:48:30.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:30 smithi063 ceph-mon[24818]: from='mgr.14100 172.21.15.63:0/1379451219' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-26T20:48:30.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:30 smithi063 ceph-mon[24818]: from='mgr.14100 172.21.15.63:0/1379451219' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-26T20:48:30.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:30 smithi063 ceph-mon[24818]: from='mgr.14100 172.21.15.63:0/1379451219' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-26T20:48:30.553 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:30 smithi063 ceph-mon[24818]: from='mgr.14100 172.21.15.63:0/1379451219' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mon metadata", "id": "smithi063"}]: dispatch 2024-03-26T20:48:30.553 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:30 smithi063 ceph-mon[24818]: from='mgr.14100 172.21.15.63:0/1379451219' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mgr metadata", "who": "smithi063.lxhujt", "id": "smithi063.lxhujt"}]: dispatch 2024-03-26T20:48:30.553 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:30 smithi063 ceph-mon[24818]: Manager daemon smithi063.lxhujt is now available 2024-03-26T20:48:30.553 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:30 smithi063 ceph-mon[24818]: from='mgr.14100 172.21.15.63:0/1379451219' entity='mgr.smithi063.lxhujt' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi063.lxhujt/mirror_snapshot_schedule"}]: dispatch 2024-03-26T20:48:30.553 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:30 smithi063 ceph-mon[24818]: from='mgr.14100 172.21.15.63:0/1379451219' entity='mgr.smithi063.lxhujt' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi063.lxhujt/trash_purge_schedule"}]: dispatch 2024-03-26T20:48:30.553 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:30 smithi063 ceph-mon[24818]: from='mgr.14100 172.21.15.63:0/1379451219' entity='mgr.smithi063.lxhujt' 2024-03-26T20:48:30.553 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:30 smithi063 ceph-mon[24818]: from='mgr.14100 172.21.15.63:0/1379451219' entity='mgr.smithi063.lxhujt' 2024-03-26T20:48:30.553 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:30 smithi063 ceph-mon[24818]: from='mgr.14100 172.21.15.63:0/1379451219' entity='mgr.smithi063.lxhujt' 2024-03-26T20:48:30.961 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: 2024-03-26T20:48:30.961 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: { 2024-03-26T20:48:30.961 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "fsid": "0ed4dd4c-ebb2-11ee-95d0-87774f69a715", 2024-03-26T20:48:30.961 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "health": { 2024-03-26T20:48:30.961 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-03-26T20:48:30.961 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "checks": {}, 2024-03-26T20:48:30.961 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "mutes": [] 2024-03-26T20:48:30.961 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: }, 2024-03-26T20:48:30.961 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-03-26T20:48:30.961 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "quorum": [ 2024-03-26T20:48:30.961 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: 0 2024-03-26T20:48:30.962 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: ], 2024-03-26T20:48:30.962 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "quorum_names": [ 2024-03-26T20:48:30.962 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "smithi063" 2024-03-26T20:48:30.962 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: ], 2024-03-26T20:48:30.962 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "quorum_age": 8, 2024-03-26T20:48:30.962 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "monmap": { 2024-03-26T20:48:30.962 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "epoch": 1, 2024-03-26T20:48:30.962 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-03-26T20:48:30.962 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "num_mons": 1 2024-03-26T20:48:30.962 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: }, 2024-03-26T20:48:30.962 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "osdmap": { 2024-03-26T20:48:30.962 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "epoch": 1, 2024-03-26T20:48:30.962 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "num_osds": 0, 2024-03-26T20:48:30.962 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-03-26T20:48:30.962 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-03-26T20:48:30.963 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-03-26T20:48:30.963 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-03-26T20:48:30.963 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-03-26T20:48:30.963 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: }, 2024-03-26T20:48:30.963 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "pgmap": { 2024-03-26T20:48:30.963 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-03-26T20:48:30.963 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-03-26T20:48:30.963 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "num_pools": 0, 2024-03-26T20:48:30.963 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "num_objects": 0, 2024-03-26T20:48:30.963 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-03-26T20:48:30.963 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-03-26T20:48:30.963 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-03-26T20:48:30.963 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "bytes_total": 0 2024-03-26T20:48:30.963 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: }, 2024-03-26T20:48:30.963 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "fsmap": { 2024-03-26T20:48:30.964 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "epoch": 1, 2024-03-26T20:48:30.964 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "by_rank": [], 2024-03-26T20:48:30.964 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "up:standby": 0 2024-03-26T20:48:30.964 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: }, 2024-03-26T20:48:30.964 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "mgrmap": { 2024-03-26T20:48:30.964 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "available": false, 2024-03-26T20:48:30.964 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-03-26T20:48:30.964 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "modules": [ 2024-03-26T20:48:30.964 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "iostat", 2024-03-26T20:48:30.964 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "nfs", 2024-03-26T20:48:30.964 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "restful" 2024-03-26T20:48:30.964 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: ], 2024-03-26T20:48:30.964 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "services": {} 2024-03-26T20:48:30.964 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: }, 2024-03-26T20:48:30.964 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "servicemap": { 2024-03-26T20:48:30.964 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "epoch": 1, 2024-03-26T20:48:30.965 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "modified": "2024-03-26T20:48:14.448691+0000", 2024-03-26T20:48:30.965 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "services": {} 2024-03-26T20:48:30.965 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: }, 2024-03-26T20:48:30.965 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "progress_events": {} 2024-03-26T20:48:30.965 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: } 2024-03-26T20:48:31.509 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:31 smithi063 ceph-mon[24818]: from='client.? 172.21.15.63:0/3038529208' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-03-26T20:48:31.510 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:31 smithi063 ceph-mon[24818]: mgrmap e3: smithi063.lxhujt(active, since 1.00566s) 2024-03-26T20:48:31.512 INFO:teuthology.orchestra.run.smithi063.stderr:mgr not available, waiting (3/15)... 2024-03-26T20:48:33.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:33 smithi063 ceph-mon[24818]: mgrmap e4: smithi063.lxhujt(active, since 2s) 2024-03-26T20:48:34.201 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: 2024-03-26T20:48:34.201 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: { 2024-03-26T20:48:34.201 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "fsid": "0ed4dd4c-ebb2-11ee-95d0-87774f69a715", 2024-03-26T20:48:34.201 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "health": { 2024-03-26T20:48:34.201 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-03-26T20:48:34.201 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "checks": {}, 2024-03-26T20:48:34.202 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "mutes": [] 2024-03-26T20:48:34.202 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: }, 2024-03-26T20:48:34.202 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-03-26T20:48:34.202 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "quorum": [ 2024-03-26T20:48:34.202 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: 0 2024-03-26T20:48:34.202 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: ], 2024-03-26T20:48:34.202 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "quorum_names": [ 2024-03-26T20:48:34.202 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "smithi063" 2024-03-26T20:48:34.202 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: ], 2024-03-26T20:48:34.202 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "quorum_age": 11, 2024-03-26T20:48:34.202 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "monmap": { 2024-03-26T20:48:34.202 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "epoch": 1, 2024-03-26T20:48:34.202 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-03-26T20:48:34.202 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "num_mons": 1 2024-03-26T20:48:34.203 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: }, 2024-03-26T20:48:34.203 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "osdmap": { 2024-03-26T20:48:34.203 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "epoch": 1, 2024-03-26T20:48:34.203 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "num_osds": 0, 2024-03-26T20:48:34.203 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-03-26T20:48:34.203 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-03-26T20:48:34.206 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-03-26T20:48:34.207 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-03-26T20:48:34.207 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-03-26T20:48:34.207 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: }, 2024-03-26T20:48:34.207 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "pgmap": { 2024-03-26T20:48:34.207 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-03-26T20:48:34.207 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-03-26T20:48:34.207 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "num_pools": 0, 2024-03-26T20:48:34.207 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "num_objects": 0, 2024-03-26T20:48:34.207 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-03-26T20:48:34.207 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-03-26T20:48:34.207 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-03-26T20:48:34.207 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "bytes_total": 0 2024-03-26T20:48:34.207 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: }, 2024-03-26T20:48:34.207 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "fsmap": { 2024-03-26T20:48:34.208 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "epoch": 1, 2024-03-26T20:48:34.208 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "by_rank": [], 2024-03-26T20:48:34.208 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "up:standby": 0 2024-03-26T20:48:34.208 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: }, 2024-03-26T20:48:34.208 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "mgrmap": { 2024-03-26T20:48:34.208 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "available": true, 2024-03-26T20:48:34.208 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-03-26T20:48:34.208 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "modules": [ 2024-03-26T20:48:34.208 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "iostat", 2024-03-26T20:48:34.208 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "nfs", 2024-03-26T20:48:34.208 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "restful" 2024-03-26T20:48:34.208 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: ], 2024-03-26T20:48:34.208 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "services": {} 2024-03-26T20:48:34.208 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: }, 2024-03-26T20:48:34.209 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "servicemap": { 2024-03-26T20:48:34.209 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "epoch": 1, 2024-03-26T20:48:34.209 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "modified": "2024-03-26T20:48:14.448691+0000", 2024-03-26T20:48:34.209 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "services": {} 2024-03-26T20:48:34.209 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: }, 2024-03-26T20:48:34.209 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "progress_events": {} 2024-03-26T20:48:34.209 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: } 2024-03-26T20:48:34.498 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:34 smithi063 ceph-mon[24818]: from='client.? 172.21.15.63:0/3025921513' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-03-26T20:48:34.835 INFO:teuthology.orchestra.run.smithi063.stderr:mgr is available 2024-03-26T20:48:35.429 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: 2024-03-26T20:48:35.429 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: [global] 2024-03-26T20:48:35.429 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: fsid = 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 2024-03-26T20:48:35.430 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-03-26T20:48:35.430 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-03-26T20:48:35.430 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-03-26T20:48:35.430 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-03-26T20:48:35.430 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: 2024-03-26T20:48:35.430 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: [mgr] 2024-03-26T20:48:35.430 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-03-26T20:48:35.430 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: 2024-03-26T20:48:35.430 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: [osd] 2024-03-26T20:48:35.430 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-03-26T20:48:35.430 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-03-26T20:48:35.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:35 smithi063 ceph-mon[24818]: from='client.? 172.21.15.63:0/326769315' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-03-26T20:48:36.000 INFO:teuthology.orchestra.run.smithi063.stderr:Enabling cephadm module... 2024-03-26T20:48:37.035 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:36 smithi063 ceph-mon[24818]: from='client.? 172.21.15.63:0/709708910' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-03-26T20:48:37.945 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:37 smithi063 ceph-mon[24818]: from='client.? 172.21.15.63:0/709708910' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-03-26T20:48:37.945 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:37 smithi063 ceph-mon[24818]: mgrmap e5: smithi063.lxhujt(active, since 6s) 2024-03-26T20:48:38.015 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: { 2024-03-26T20:48:38.015 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "epoch": 5, 2024-03-26T20:48:38.015 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "available": true, 2024-03-26T20:48:38.015 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "active_name": "smithi063.lxhujt", 2024-03-26T20:48:38.015 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "num_standby": 0 2024-03-26T20:48:38.015 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: } 2024-03-26T20:48:38.670 INFO:teuthology.orchestra.run.smithi063.stderr:Waiting for the mgr to restart... 2024-03-26T20:48:38.670 INFO:teuthology.orchestra.run.smithi063.stderr:Waiting for mgr epoch 5... 2024-03-26T20:48:38.813 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:38 smithi063 ceph-mon[24818]: from='client.? 172.21.15.63:0/3068668284' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-03-26T20:48:43.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:43 smithi063 ceph-mon[24818]: Active manager daemon smithi063.lxhujt restarted 2024-03-26T20:48:43.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:43 smithi063 ceph-mon[24818]: Activating manager daemon smithi063.lxhujt 2024-03-26T20:48:43.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:43 smithi063 ceph-mon[24818]: osdmap e2: 0 total, 0 up, 0 in 2024-03-26T20:48:44.282 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: { 2024-03-26T20:48:44.282 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "mgrmap_epoch": 7, 2024-03-26T20:48:44.282 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "initialized": true 2024-03-26T20:48:44.283 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: } 2024-03-26T20:48:44.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:44 smithi063 ceph-mon[24818]: mgrmap e6: smithi063.lxhujt(active, starting, since 0.5057s) 2024-03-26T20:48:44.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:44 smithi063 ceph-mon[24818]: from='mgr.14120 172.21.15.63:0/699793752' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mon metadata", "id": "smithi063"}]: dispatch 2024-03-26T20:48:44.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:44 smithi063 ceph-mon[24818]: from='mgr.14120 172.21.15.63:0/699793752' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mgr metadata", "who": "smithi063.lxhujt", "id": "smithi063.lxhujt"}]: dispatch 2024-03-26T20:48:44.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:44 smithi063 ceph-mon[24818]: from='mgr.14120 172.21.15.63:0/699793752' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-26T20:48:44.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:44 smithi063 ceph-mon[24818]: from='mgr.14120 172.21.15.63:0/699793752' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-26T20:48:44.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:44 smithi063 ceph-mon[24818]: from='mgr.14120 172.21.15.63:0/699793752' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-26T20:48:44.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:44 smithi063 ceph-mon[24818]: Manager daemon smithi063.lxhujt is now available 2024-03-26T20:48:44.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:44 smithi063 ceph-mon[24818]: from='mgr.14120 172.21.15.63:0/699793752' entity='mgr.smithi063.lxhujt' 2024-03-26T20:48:44.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:44 smithi063 ceph-mon[24818]: from='mgr.14120 172.21.15.63:0/699793752' entity='mgr.smithi063.lxhujt' 2024-03-26T20:48:44.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:44 smithi063 ceph-mon[24818]: from='mgr.14120 172.21.15.63:0/699793752' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T20:48:44.553 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:44 smithi063 ceph-mon[24818]: from='mgr.14120 172.21.15.63:0/699793752' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T20:48:44.553 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:44 smithi063 ceph-mon[24818]: from='mgr.14120 172.21.15.63:0/699793752' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T20:48:44.553 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:44 smithi063 ceph-mon[24818]: from='mgr.14120 172.21.15.63:0/699793752' entity='mgr.smithi063.lxhujt' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi063.lxhujt/mirror_snapshot_schedule"}]: dispatch 2024-03-26T20:48:44.553 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:44 smithi063 ceph-mon[24818]: from='mgr.14120 172.21.15.63:0/699793752' entity='mgr.smithi063.lxhujt' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi063.lxhujt/trash_purge_schedule"}]: dispatch 2024-03-26T20:48:44.808 INFO:teuthology.orchestra.run.smithi063.stderr:mgr epoch 5 is available 2024-03-26T20:48:44.808 INFO:teuthology.orchestra.run.smithi063.stderr:Setting orchestrator backend to cephadm... 2024-03-26T20:48:45.440 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:45 smithi063 ceph-mon[24818]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-03-26T20:48:45.440 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:45 smithi063 ceph-mon[24818]: mgrmap e7: smithi063.lxhujt(active, since 1.51226s) 2024-03-26T20:48:45.440 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:45 smithi063 ceph-mon[24818]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-03-26T20:48:45.440 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:45 smithi063 ceph-mon[24818]: from='mgr.14120 172.21.15.63:0/699793752' entity='mgr.smithi063.lxhujt' 2024-03-26T20:48:45.440 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:45 smithi063 ceph-mon[24818]: from='mgr.14120 172.21.15.63:0/699793752' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T20:48:46.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:46 smithi063 ceph-mon[24818]: [26/Mar/2024:20:48:44] ENGINE Bus STARTING 2024-03-26T20:48:46.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:46 smithi063 ceph-mon[24818]: [26/Mar/2024:20:48:44] ENGINE Serving on https://172.21.15.63:7150 2024-03-26T20:48:46.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:46 smithi063 ceph-mon[24818]: [26/Mar/2024:20:48:44] ENGINE Bus STARTED 2024-03-26T20:48:46.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:46 smithi063 ceph-mon[24818]: mgrmap e8: smithi063.lxhujt(active, since 2s) 2024-03-26T20:48:46.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:46 smithi063 ceph-mon[24818]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:48:46.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:46 smithi063 ceph-mon[24818]: from='mgr.14120 172.21.15.63:0/699793752' entity='mgr.smithi063.lxhujt' 2024-03-26T20:48:46.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:46 smithi063 ceph-mon[24818]: from='mgr.14120 172.21.15.63:0/699793752' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T20:48:46.558 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: value unchanged 2024-03-26T20:48:47.181 INFO:teuthology.orchestra.run.smithi063.stderr:Generating ssh key... 2024-03-26T20:48:47.520 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:47 smithi063 ceph-mon[24818]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:48:49.188 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:48 smithi063 ceph-mon[24818]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:48:49.188 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:48 smithi063 ceph-mon[24818]: Generating ssh key... 2024-03-26T20:48:49.188 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:48 smithi063 ceph-mon[24818]: from='mgr.14120 172.21.15.63:0/699793752' entity='mgr.smithi063.lxhujt' 2024-03-26T20:48:49.188 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:48 smithi063 ceph-mon[24818]: from='mgr.14120 172.21.15.63:0/699793752' entity='mgr.smithi063.lxhujt' 2024-03-26T20:48:49.188 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC78pPXoeYDrJnhEvqb0nki2f3cWJxteEg23Hi8sEyHy7d943J5NFFurha7Zi2CujVmlADuNVRnGK5hqr43YayZyu7EtsJvoskT7auNDi0yHHvfsYy40RUsPWH9ODbGZUi4jPbRWVyTvM4wAeHVAMBCQVJkmhh22WGSMZmfsK6qA0EAT677orKKWV+IdCvZysmEK55OEV2Mmue7GDzSQWg5Ku5WVcXIZhZ/dcHpoHimpksicXLPmAk62d5yyfcTTwozi19o9CoWer1x4rL9n11EK9Ro2plen7mcRd4lgZq9mLSGPmBqxdPvvhkBaOdJ4glAJUF8asEdxwLnEr/Ry5aqmDzqzHlQ+UQotzZELywIyOvnnRBWPSlBWkfU6CHes8pOKa63O4XcXUetdTy2QxTMTymvN3t+HmsfCoejz1Gpt9zkqYoERcQk0CGu6NRdsR2lK1VxrKP1q1OUlU+tPA37Tw5qJOu5B3dJaTkTyHQ8BKODYtZguX7r9Z7hgyXFi4s= ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715 2024-03-26T20:48:49.766 INFO:teuthology.orchestra.run.smithi063.stderr:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-03-26T20:48:49.766 INFO:teuthology.orchestra.run.smithi063.stderr:Adding key to root@localhost authorized_keys... 2024-03-26T20:48:49.766 INFO:teuthology.orchestra.run.smithi063.stderr:Adding host smithi063... 2024-03-26T20:48:50.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:49 smithi063 ceph-mon[24818]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:48:51.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:50 smithi063 ceph-mon[24818]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi063", "addr": "172.21.15.63", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:48:51.383 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: Added host 'smithi063' with addr '172.21.15.63' 2024-03-26T20:48:51.989 INFO:teuthology.orchestra.run.smithi063.stderr:Deploying mon service with default placement... 2024-03-26T20:48:52.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:52 smithi063 ceph-mon[24818]: Deploying cephadm binary to smithi063 2024-03-26T20:48:52.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:52 smithi063 ceph-mon[24818]: from='mgr.14120 172.21.15.63:0/699793752' entity='mgr.smithi063.lxhujt' 2024-03-26T20:48:52.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:52 smithi063 ceph-mon[24818]: Added host smithi063 2024-03-26T20:48:52.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:52 smithi063 ceph-mon[24818]: from='mgr.14120 172.21.15.63:0/699793752' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T20:48:52.585 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: Scheduled mon update... 2024-03-26T20:48:53.498 INFO:teuthology.orchestra.run.smithi063.stderr:Deploying mgr service with default placement... 2024-03-26T20:48:53.624 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:53 smithi063 ceph-mon[24818]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:48:53.625 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:53 smithi063 ceph-mon[24818]: Saving service mon spec with placement count:5 2024-03-26T20:48:53.625 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:53 smithi063 ceph-mon[24818]: from='mgr.14120 172.21.15.63:0/699793752' entity='mgr.smithi063.lxhujt' 2024-03-26T20:48:54.135 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: Scheduled mgr update... 2024-03-26T20:48:54.739 INFO:teuthology.orchestra.run.smithi063.stderr:Deploying crash service with default placement... 2024-03-26T20:48:55.449 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:55 smithi063 ceph-mon[24818]: from='client.14144 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:48:55.449 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:55 smithi063 ceph-mon[24818]: Saving service mgr spec with placement count:2 2024-03-26T20:48:55.449 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:55 smithi063 ceph-mon[24818]: from='mgr.14120 172.21.15.63:0/699793752' entity='mgr.smithi063.lxhujt' 2024-03-26T20:48:56.123 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: Scheduled crash update... 2024-03-26T20:48:56.674 INFO:teuthology.orchestra.run.smithi063.stderr:Deploying prometheus service with default placement... 2024-03-26T20:48:56.795 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:56 smithi063 ceph-mon[24818]: from='mgr.14120 172.21.15.63:0/699793752' entity='mgr.smithi063.lxhujt' 2024-03-26T20:48:56.795 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:56 smithi063 ceph-mon[24818]: from='mgr.14120 172.21.15.63:0/699793752' entity='mgr.smithi063.lxhujt' 2024-03-26T20:48:56.795 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:56 smithi063 ceph-mon[24818]: from='mgr.14120 172.21.15.63:0/699793752' entity='mgr.smithi063.lxhujt' 2024-03-26T20:48:57.273 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: Scheduled prometheus update... 2024-03-26T20:48:57.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:57 smithi063 ceph-mon[24818]: from='client.14146 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "crash", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:48:57.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:57 smithi063 ceph-mon[24818]: Saving service crash spec with placement * 2024-03-26T20:48:57.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:57 smithi063 ceph-mon[24818]: from='mgr.14120 172.21.15.63:0/699793752' entity='mgr.smithi063.lxhujt' 2024-03-26T20:48:58.283 INFO:teuthology.orchestra.run.smithi063.stderr:Deploying grafana service with default placement... 2024-03-26T20:48:58.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:58 smithi063 ceph-mon[24818]: from='client.14148 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:48:58.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:58 smithi063 ceph-mon[24818]: Saving service prometheus spec with placement count:1 2024-03-26T20:48:58.925 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: Scheduled grafana update... 2024-03-26T20:48:59.531 INFO:teuthology.orchestra.run.smithi063.stderr:Deploying node-exporter service with default placement... 2024-03-26T20:49:00.052 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:59 smithi063 ceph-mon[24818]: from='mgr.14120 172.21.15.63:0/699793752' entity='mgr.smithi063.lxhujt' 2024-03-26T20:49:00.052 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:59 smithi063 ceph-mon[24818]: from='client.14150 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:49:00.052 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:59 smithi063 ceph-mon[24818]: Saving service grafana spec with placement count:1 2024-03-26T20:49:00.052 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:59 smithi063 ceph-mon[24818]: from='mgr.14120 172.21.15.63:0/699793752' entity='mgr.smithi063.lxhujt' 2024-03-26T20:49:00.052 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:48:59 smithi063 ceph-mon[24818]: from='mgr.14120 172.21.15.63:0/699793752' entity='mgr.smithi063.lxhujt' 2024-03-26T20:49:00.201 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: Scheduled node-exporter update... 2024-03-26T20:49:01.124 INFO:teuthology.orchestra.run.smithi063.stderr:Deploying alertmanager service with default placement... 2024-03-26T20:49:01.374 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:01 smithi063 ceph-mon[24818]: from='client.14152 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:49:01.374 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:01 smithi063 ceph-mon[24818]: Saving service node-exporter spec with placement * 2024-03-26T20:49:01.374 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:01 smithi063 ceph-mon[24818]: from='mgr.14120 172.21.15.63:0/699793752' entity='mgr.smithi063.lxhujt' 2024-03-26T20:49:01.783 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: Scheduled alertmanager update... 2024-03-26T20:49:02.906 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:02 smithi063 ceph-mon[24818]: from='client.14154 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:49:02.906 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:02 smithi063 ceph-mon[24818]: Saving service alertmanager spec with placement count:1 2024-03-26T20:49:02.906 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:02 smithi063 ceph-mon[24818]: from='mgr.14120 172.21.15.63:0/699793752' entity='mgr.smithi063.lxhujt' 2024-03-26T20:49:04.551 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:04 smithi063 ceph-mon[24818]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-26T20:49:04.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:04 smithi063 ceph-mon[24818]: from='client.? 172.21.15.63:0/73703493' entity='client.admin' 2024-03-26T20:49:05.170 INFO:teuthology.orchestra.run.smithi063.stderr:Enabling the dashboard module... 2024-03-26T20:49:05.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:05 smithi063 ceph-mon[24818]: from='client.? 172.21.15.63:0/1522810614' entity='client.admin' 2024-03-26T20:49:06.848 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:06 smithi063 ceph-mon[24818]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-26T20:49:06.849 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:06 smithi063 ceph-mon[24818]: from='client.? 172.21.15.63:0/3042578810' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-03-26T20:49:07.793 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:07 smithi063 ceph-mon[24818]: from='client.? 172.21.15.63:0/3042578810' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-03-26T20:49:07.793 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:07 smithi063 ceph-mon[24818]: mgrmap e9: smithi063.lxhujt(active, since 23s) 2024-03-26T20:49:07.822 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: { 2024-03-26T20:49:07.822 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "epoch": 9, 2024-03-26T20:49:07.822 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "available": true, 2024-03-26T20:49:07.822 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "active_name": "smithi063.lxhujt", 2024-03-26T20:49:07.822 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "num_standby": 0 2024-03-26T20:49:07.823 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: } 2024-03-26T20:49:08.450 INFO:teuthology.orchestra.run.smithi063.stderr:Waiting for the mgr to restart... 2024-03-26T20:49:08.450 INFO:teuthology.orchestra.run.smithi063.stderr:Waiting for mgr epoch 9... 2024-03-26T20:49:08.659 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:08 smithi063 ceph-mon[24818]: from='client.? 172.21.15.63:0/2779797314' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-03-26T20:49:13.529 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:13 smithi063 ceph-mon[24818]: Active manager daemon smithi063.lxhujt restarted 2024-03-26T20:49:13.529 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:13 smithi063 ceph-mon[24818]: Activating manager daemon smithi063.lxhujt 2024-03-26T20:49:13.530 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:13 smithi063 ceph-mon[24818]: osdmap e3: 0 total, 0 up, 0 in 2024-03-26T20:49:14.286 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: { 2024-03-26T20:49:14.286 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:14 smithi063 ceph-mon[24818]: mgrmap e10: smithi063.lxhujt(active, starting, since 0.692439s) 2024-03-26T20:49:14.286 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:14 smithi063 ceph-mon[24818]: from='mgr.14162 172.21.15.63:0/3544862618' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mon metadata", "id": "smithi063"}]: dispatch 2024-03-26T20:49:14.286 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:14 smithi063 ceph-mon[24818]: from='mgr.14162 172.21.15.63:0/3544862618' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mgr metadata", "who": "smithi063.lxhujt", "id": "smithi063.lxhujt"}]: dispatch 2024-03-26T20:49:14.286 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:14 smithi063 ceph-mon[24818]: from='mgr.14162 172.21.15.63:0/3544862618' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-26T20:49:14.287 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:14 smithi063 ceph-mon[24818]: from='mgr.14162 172.21.15.63:0/3544862618' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-26T20:49:14.287 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:14 smithi063 ceph-mon[24818]: from='mgr.14162 172.21.15.63:0/3544862618' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-26T20:49:14.287 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:14 smithi063 ceph-mon[24818]: Manager daemon smithi063.lxhujt is now available 2024-03-26T20:49:14.287 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:14 smithi063 ceph-mon[24818]: from='mgr.14162 172.21.15.63:0/3544862618' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T20:49:14.287 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:14 smithi063 ceph-mon[24818]: from='mgr.14162 172.21.15.63:0/3544862618' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T20:49:14.287 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:14 smithi063 ceph-mon[24818]: from='mgr.14162 172.21.15.63:0/3544862618' entity='mgr.smithi063.lxhujt' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi063.lxhujt/mirror_snapshot_schedule"}]: dispatch 2024-03-26T20:49:14.287 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:14 smithi063 ceph-mon[24818]: from='mgr.14162 172.21.15.63:0/3544862618' entity='mgr.smithi063.lxhujt' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi063.lxhujt/trash_purge_schedule"}]: dispatch 2024-03-26T20:49:14.287 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "mgrmap_epoch": 11, 2024-03-26T20:49:14.288 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: "initialized": true 2024-03-26T20:49:14.288 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: } 2024-03-26T20:49:15.077 INFO:teuthology.orchestra.run.smithi063.stderr:mgr epoch 9 is available 2024-03-26T20:49:15.077 INFO:teuthology.orchestra.run.smithi063.stderr:Generating a dashboard self-signed certificate... 2024-03-26T20:49:15.327 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:15 smithi063 ceph-mon[24818]: mgrmap e11: smithi063.lxhujt(active, since 1.69676s) 2024-03-26T20:49:15.328 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:15 smithi063 ceph-mon[24818]: [26/Mar/2024:20:49:14] ENGINE Bus STARTING 2024-03-26T20:49:15.328 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:15 smithi063 ceph-mon[24818]: from='mgr.14162 172.21.15.63:0/3544862618' entity='mgr.smithi063.lxhujt' 2024-03-26T20:49:15.328 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:15 smithi063 ceph-mon[24818]: [26/Mar/2024:20:49:14] ENGINE Serving on https://172.21.15.63:7150 2024-03-26T20:49:15.328 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:15 smithi063 ceph-mon[24818]: [26/Mar/2024:20:49:14] ENGINE Bus STARTED 2024-03-26T20:49:15.328 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:15 smithi063 ceph-mon[24818]: from='mgr.14162 172.21.15.63:0/3544862618' entity='mgr.smithi063.lxhujt' 2024-03-26T20:49:15.909 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: Self-signed certificate created 2024-03-26T20:49:16.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:16 smithi063 ceph-mon[24818]: mgrmap e12: smithi063.lxhujt(active, since 2s) 2024-03-26T20:49:16.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:16 smithi063 ceph-mon[24818]: from='mgr.14162 172.21.15.63:0/3544862618' entity='mgr.smithi063.lxhujt' 2024-03-26T20:49:16.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:16 smithi063 ceph-mon[24818]: from='mgr.14162 172.21.15.63:0/3544862618' entity='mgr.smithi063.lxhujt' 2024-03-26T20:49:16.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:16 smithi063 ceph-mon[24818]: from='mgr.14162 172.21.15.63:0/3544862618' entity='mgr.smithi063.lxhujt' 2024-03-26T20:49:16.599 INFO:teuthology.orchestra.run.smithi063.stderr:Creating initial admin user... 2024-03-26T20:49:17.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:17 smithi063 ceph-mon[24818]: from='client.14174 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:49:18.797 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: {"username": "admin", "password": "$2b$12$zYWqSvFZFey5oZG/5vGcuewcOkI/yCj6W9SVhYVM4pWMObcOnTJi2", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1711486158, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-03-26T20:49:19.052 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:18 smithi063 ceph-mon[24818]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:49:19.052 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:18 smithi063 ceph-mon[24818]: mgrmap e13: smithi063.lxhujt(active, since 6s) 2024-03-26T20:49:19.421 INFO:teuthology.orchestra.run.smithi063.stderr:Fetching dashboard port number... 2024-03-26T20:49:19.988 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:19 smithi063 ceph-mon[24818]: from='mgr.14162 172.21.15.63:0/3544862618' entity='mgr.smithi063.lxhujt' 2024-03-26T20:49:20.217 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: 8443 2024-03-26T20:49:20.916 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:20 smithi063 ceph-mon[24818]: from='client.? 172.21.15.63:0/2921283507' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-03-26T20:49:20.929 INFO:teuthology.orchestra.run.smithi063.stderr:systemctl: disabled 2024-03-26T20:49:20.936 INFO:teuthology.orchestra.run.smithi063.stderr:systemctl: inactive 2024-03-26T20:49:20.936 INFO:teuthology.orchestra.run.smithi063.stderr:firewalld.service is not enabled 2024-03-26T20:49:20.937 INFO:teuthology.orchestra.run.smithi063.stderr:Not possible to open ports <[8443]>. firewalld.service is not available 2024-03-26T20:49:20.939 INFO:teuthology.orchestra.run.smithi063.stderr:Ceph Dashboard is now available at: 2024-03-26T20:49:20.939 INFO:teuthology.orchestra.run.smithi063.stderr: 2024-03-26T20:49:20.939 INFO:teuthology.orchestra.run.smithi063.stderr: URL: https://smithi063.front.sepia.ceph.com:8443/ 2024-03-26T20:49:20.939 INFO:teuthology.orchestra.run.smithi063.stderr: User: admin 2024-03-26T20:49:20.939 INFO:teuthology.orchestra.run.smithi063.stderr: Password: 7hqw1scjfo 2024-03-26T20:49:20.939 INFO:teuthology.orchestra.run.smithi063.stderr: 2024-03-26T20:49:20.939 INFO:teuthology.orchestra.run.smithi063.stderr:Enabling autotune for osd_memory_target 2024-03-26T20:49:22.899 INFO:teuthology.orchestra.run.smithi063.stderr:/usr/bin/ceph: set mgr/dashboard/cluster/status 2024-03-26T20:49:23.525 INFO:teuthology.orchestra.run.smithi063.stderr:You can access the Ceph CLI with: 2024-03-26T20:49:23.525 INFO:teuthology.orchestra.run.smithi063.stderr: 2024-03-26T20:49:23.525 INFO:teuthology.orchestra.run.smithi063.stderr: sudo /home/ubuntu/cephtest/cephadm shell --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-03-26T20:49:23.525 INFO:teuthology.orchestra.run.smithi063.stderr: 2024-03-26T20:49:23.525 INFO:teuthology.orchestra.run.smithi063.stderr:Please consider enabling telemetry to help improve Ceph: 2024-03-26T20:49:23.525 INFO:teuthology.orchestra.run.smithi063.stderr: 2024-03-26T20:49:23.525 INFO:teuthology.orchestra.run.smithi063.stderr: ceph telemetry on 2024-03-26T20:49:23.525 INFO:teuthology.orchestra.run.smithi063.stderr: 2024-03-26T20:49:23.525 INFO:teuthology.orchestra.run.smithi063.stderr:For more information see: 2024-03-26T20:49:23.525 INFO:teuthology.orchestra.run.smithi063.stderr: 2024-03-26T20:49:23.526 INFO:teuthology.orchestra.run.smithi063.stderr: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-03-26T20:49:23.526 INFO:teuthology.orchestra.run.smithi063.stderr: 2024-03-26T20:49:23.526 INFO:teuthology.orchestra.run.smithi063.stderr:Bootstrap complete. 2024-03-26T20:49:23.562 INFO:tasks.cephadm:Fetching config... 2024-03-26T20:49:23.562 DEBUG:teuthology.orchestra.run.smithi063:> set -ex 2024-03-26T20:49:23.563 DEBUG:teuthology.orchestra.run.smithi063:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-03-26T20:49:23.578 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-03-26T20:49:23.578 DEBUG:teuthology.orchestra.run.smithi063:> set -ex 2024-03-26T20:49:23.578 DEBUG:teuthology.orchestra.run.smithi063:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-03-26T20:49:23.633 INFO:tasks.cephadm:Fetching mon keyring... 2024-03-26T20:49:23.633 DEBUG:teuthology.orchestra.run.smithi063:> set -ex 2024-03-26T20:49:23.633 DEBUG:teuthology.orchestra.run.smithi063:> sudo dd if=/var/lib/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/mon.smithi063/keyring of=/dev/stdout 2024-03-26T20:49:23.700 INFO:tasks.cephadm:Fetching pub ssh key... 2024-03-26T20:49:23.700 DEBUG:teuthology.orchestra.run.smithi063:> set -ex 2024-03-26T20:49:23.700 DEBUG:teuthology.orchestra.run.smithi063:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-03-26T20:49:23.755 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-03-26T20:49:23.755 DEBUG:teuthology.orchestra.run.smithi063:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC78pPXoeYDrJnhEvqb0nki2f3cWJxteEg23Hi8sEyHy7d943J5NFFurha7Zi2CujVmlADuNVRnGK5hqr43YayZyu7EtsJvoskT7auNDi0yHHvfsYy40RUsPWH9ODbGZUi4jPbRWVyTvM4wAeHVAMBCQVJkmhh22WGSMZmfsK6qA0EAT677orKKWV+IdCvZysmEK55OEV2Mmue7GDzSQWg5Ku5WVcXIZhZ/dcHpoHimpksicXLPmAk62d5yyfcTTwozi19o9CoWer1x4rL9n11EK9Ro2plen7mcRd4lgZq9mLSGPmBqxdPvvhkBaOdJ4glAJUF8asEdxwLnEr/Ry5aqmDzqzHlQ+UQotzZELywIyOvnnRBWPSlBWkfU6CHes8pOKa63O4XcXUetdTy2QxTMTymvN3t+HmsfCoejz1Gpt9zkqYoERcQk0CGu6NRdsR2lK1VxrKP1q1OUlU+tPA37Tw5qJOu5B3dJaTkTyHQ8BKODYtZguX7r9Z7hgyXFi4s= ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-03-26T20:49:23.840 INFO:teuthology.orchestra.run.smithi063.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC78pPXoeYDrJnhEvqb0nki2f3cWJxteEg23Hi8sEyHy7d943J5NFFurha7Zi2CujVmlADuNVRnGK5hqr43YayZyu7EtsJvoskT7auNDi0yHHvfsYy40RUsPWH9ODbGZUi4jPbRWVyTvM4wAeHVAMBCQVJkmhh22WGSMZmfsK6qA0EAT677orKKWV+IdCvZysmEK55OEV2Mmue7GDzSQWg5Ku5WVcXIZhZ/dcHpoHimpksicXLPmAk62d5yyfcTTwozi19o9CoWer1x4rL9n11EK9Ro2plen7mcRd4lgZq9mLSGPmBqxdPvvhkBaOdJ4glAJUF8asEdxwLnEr/Ry5aqmDzqzHlQ+UQotzZELywIyOvnnRBWPSlBWkfU6CHes8pOKa63O4XcXUetdTy2QxTMTymvN3t+HmsfCoejz1Gpt9zkqYoERcQk0CGu6NRdsR2lK1VxrKP1q1OUlU+tPA37Tw5qJOu5B3dJaTkTyHQ8BKODYtZguX7r9Z7hgyXFi4s= ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715 2024-03-26T20:49:23.855 DEBUG:teuthology.orchestra.run.smithi174:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC78pPXoeYDrJnhEvqb0nki2f3cWJxteEg23Hi8sEyHy7d943J5NFFurha7Zi2CujVmlADuNVRnGK5hqr43YayZyu7EtsJvoskT7auNDi0yHHvfsYy40RUsPWH9ODbGZUi4jPbRWVyTvM4wAeHVAMBCQVJkmhh22WGSMZmfsK6qA0EAT677orKKWV+IdCvZysmEK55OEV2Mmue7GDzSQWg5Ku5WVcXIZhZ/dcHpoHimpksicXLPmAk62d5yyfcTTwozi19o9CoWer1x4rL9n11EK9Ro2plen7mcRd4lgZq9mLSGPmBqxdPvvhkBaOdJ4glAJUF8asEdxwLnEr/Ry5aqmDzqzHlQ+UQotzZELywIyOvnnRBWPSlBWkfU6CHes8pOKa63O4XcXUetdTy2QxTMTymvN3t+HmsfCoejz1Gpt9zkqYoERcQk0CGu6NRdsR2lK1VxrKP1q1OUlU+tPA37Tw5qJOu5B3dJaTkTyHQ8BKODYtZguX7r9Z7hgyXFi4s= ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-03-26T20:49:23.904 INFO:teuthology.orchestra.run.smithi174.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC78pPXoeYDrJnhEvqb0nki2f3cWJxteEg23Hi8sEyHy7d943J5NFFurha7Zi2CujVmlADuNVRnGK5hqr43YayZyu7EtsJvoskT7auNDi0yHHvfsYy40RUsPWH9ODbGZUi4jPbRWVyTvM4wAeHVAMBCQVJkmhh22WGSMZmfsK6qA0EAT677orKKWV+IdCvZysmEK55OEV2Mmue7GDzSQWg5Ku5WVcXIZhZ/dcHpoHimpksicXLPmAk62d5yyfcTTwozi19o9CoWer1x4rL9n11EK9Ro2plen7mcRd4lgZq9mLSGPmBqxdPvvhkBaOdJ4glAJUF8asEdxwLnEr/Ry5aqmDzqzHlQ+UQotzZELywIyOvnnRBWPSlBWkfU6CHes8pOKa63O4XcXUetdTy2QxTMTymvN3t+HmsfCoejz1Gpt9zkqYoERcQk0CGu6NRdsR2lK1VxrKP1q1OUlU+tPA37Tw5qJOu5B3dJaTkTyHQ8BKODYtZguX7r9Z7hgyXFi4s= ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715 2024-03-26T20:49:23.920 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-03-26T20:49:24.100 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:23 smithi063 ceph-mon[24818]: from='client.? 172.21.15.63:0/373010095' entity='client.admin' 2024-03-26T20:49:27.631 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-03-26T20:49:27.632 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-03-26T20:49:28.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:28 smithi063 ceph-mon[24818]: from='client.? 172.21.15.63:0/4099328193' entity='client.admin' 2024-03-26T20:49:29.328 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:29 smithi063 ceph-mon[24818]: from='mgr.14162 172.21.15.63:0/3544862618' entity='mgr.smithi063.lxhujt' 2024-03-26T20:49:29.328 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:29 smithi063 ceph-mon[24818]: from='mgr.14162 172.21.15.63:0/3544862618' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "who": "osd/host:smithi063", "name": "osd_memory_target"}]: dispatch 2024-03-26T20:49:29.328 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:29 smithi063 ceph-mon[24818]: from='mgr.14162 172.21.15.63:0/3544862618' entity='mgr.smithi063.lxhujt' 2024-03-26T20:49:29.328 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:29 smithi063 ceph-mon[24818]: Deploying daemon alertmanager.smithi063 on smithi063 2024-03-26T20:49:30.781 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi174 2024-03-26T20:49:30.781 DEBUG:teuthology.orchestra.run.smithi174:> set -ex 2024-03-26T20:49:30.781 DEBUG:teuthology.orchestra.run.smithi174:> dd of=/etc/ceph/ceph.conf 2024-03-26T20:49:30.800 DEBUG:teuthology.orchestra.run.smithi174:> set -ex 2024-03-26T20:49:30.800 DEBUG:teuthology.orchestra.run.smithi174:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-03-26T20:49:30.857 INFO:tasks.cephadm:Adding host smithi174 to orchestrator... 2024-03-26T20:49:30.858 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -- ceph orch host add smithi174 2024-03-26T20:49:31.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:30 smithi063 ceph-mon[24818]: from='client.14186 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:49:31.052 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:30 smithi063 ceph-mon[24818]: from='mgr.14162 172.21.15.63:0/3544862618' entity='mgr.smithi063.lxhujt' 2024-03-26T20:49:34.428 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:34 smithi063 ceph-mon[24818]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-26T20:49:35.321 INFO:teuthology.orchestra.run.smithi063.stdout:Added host 'smithi174' with addr '172.21.15.174' 2024-03-26T20:49:35.527 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:35 smithi063 ceph-mon[24818]: from='client.14188 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi174", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:49:35.527 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:35 smithi063 ceph-mon[24818]: from='mgr.14162 172.21.15.63:0/3544862618' entity='mgr.smithi063.lxhujt' 2024-03-26T20:49:36.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:36 smithi063 ceph-mon[24818]: Deploying cephadm binary to smithi174 2024-03-26T20:49:36.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:36 smithi063 ceph-mon[24818]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-26T20:49:36.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:36 smithi063 ceph-mon[24818]: Added host smithi174 2024-03-26T20:49:36.956 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -- ceph orch host ls --format=json 2024-03-26T20:49:38.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:38 smithi063 ceph-mon[24818]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-26T20:49:39.911 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-03-26T20:49:39.911 INFO:teuthology.orchestra.run.smithi063.stdout:[{"addr": "172.21.15.63", "hostname": "smithi063", "labels": [], "status": ""}, {"addr": "172.21.15.174", "hostname": "smithi174", "labels": [], "status": ""}] 2024-03-26T20:49:40.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:40 smithi063 ceph-mon[24818]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-26T20:49:40.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:40 smithi063 ceph-mon[24818]: from='mgr.14162 172.21.15.63:0/3544862618' entity='mgr.smithi063.lxhujt' 2024-03-26T20:49:40.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:40 smithi063 ceph-mon[24818]: from='mgr.14162 172.21.15.63:0/3544862618' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi063", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-03-26T20:49:40.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:40 smithi063 ceph-mon[24818]: from='mgr.14162 172.21.15.63:0/3544862618' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi063", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-03-26T20:49:40.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:40 smithi063 ceph-mon[24818]: from='mgr.14162 172.21.15.63:0/3544862618' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:49:40.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:40 smithi063 ceph-mon[24818]: Deploying daemon crash.smithi063 on smithi063 2024-03-26T20:49:41.561 INFO:tasks.cephadm:Setting crush tunables to default 2024-03-26T20:49:41.561 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -- ceph osd crush tunables default 2024-03-26T20:49:41.773 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:41 smithi063 ceph-mon[24818]: from='client.14190 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-26T20:49:42.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:42 smithi063 ceph-mon[24818]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-26T20:49:42.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:42 smithi063 ceph-mon[24818]: from='mgr.14162 172.21.15.63:0/3544862618' entity='mgr.smithi063.lxhujt' 2024-03-26T20:49:42.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:42 smithi063 ceph-mon[24818]: from='mgr.14162 172.21.15.63:0/3544862618' entity='mgr.smithi063.lxhujt' 2024-03-26T20:49:42.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:42 smithi063 ceph-mon[24818]: from='mgr.14162 172.21.15.63:0/3544862618' entity='mgr.smithi063.lxhujt' 2024-03-26T20:49:42.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:42 smithi063 ceph-mon[24818]: from='mgr.14162 172.21.15.63:0/3544862618' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-03-26T20:49:42.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:42 smithi063 ceph-mon[24818]: from='mgr.14162 172.21.15.63:0/3544862618' entity='mgr.smithi063.lxhujt' 2024-03-26T20:49:43.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:43 smithi063 ceph-mon[24818]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-03-26T20:49:43.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:43 smithi063 ceph-mon[24818]: Deploying daemon grafana.smithi063 on smithi063 2024-03-26T20:49:43.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:43 smithi063 ceph-mon[24818]: from='mgr.14162 172.21.15.63:0/3544862618' entity='mgr.smithi063.lxhujt' 2024-03-26T20:49:44.536 INFO:teuthology.orchestra.run.smithi063.stderr:adjusted tunables profile to default 2024-03-26T20:49:44.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:44 smithi063 ceph-mon[24818]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-26T20:49:44.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:44 smithi063 ceph-mon[24818]: from='client.? 172.21.15.63:0/2475152064' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-03-26T20:49:45.196 INFO:tasks.cephadm:Adding mon.smithi063 on smithi063 2024-03-26T20:49:45.196 INFO:tasks.cephadm:Adding mon.smithi174 on smithi174 2024-03-26T20:49:45.197 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -- ceph orch apply mon '2;smithi063:172.21.15.63=smithi063;smithi174:172.21.15.174=smithi174' 2024-03-26T20:49:45.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:45 smithi063 ceph-mon[24818]: from='client.? 172.21.15.63:0/2475152064' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-03-26T20:49:45.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:45 smithi063 ceph-mon[24818]: osdmap e4: 0 total, 0 up, 0 in 2024-03-26T20:49:46.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:46 smithi063 ceph-mon[24818]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-26T20:49:47.041 INFO:teuthology.orchestra.run.smithi174.stdout:Scheduled mon update... 2024-03-26T20:49:47.554 DEBUG:teuthology.orchestra.run.smithi174:mon.smithi174> sudo journalctl -f -n 0 -u ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715@mon.smithi174.service 2024-03-26T20:49:47.557 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-03-26T20:49:47.557 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -- ceph mon dump -f json 2024-03-26T20:49:48.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:48 smithi063 ceph-mon[24818]: from='client.14194 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "2;smithi063:172.21.15.63=smithi063;smithi174:172.21.15.174=smithi174", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:49:48.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:48 smithi063 ceph-mon[24818]: Saving service mon spec with placement smithi063:172.21.15.63=smithi063;smithi174:172.21.15.174=smithi174;count:2 2024-03-26T20:49:48.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:48 smithi063 ceph-mon[24818]: from='mgr.14162 172.21.15.63:0/3544862618' entity='mgr.smithi063.lxhujt' 2024-03-26T20:49:48.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:48 smithi063 ceph-mon[24818]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-26T20:49:49.336 INFO:teuthology.orchestra.run.smithi174.stdout: 2024-03-26T20:49:49.336 INFO:teuthology.orchestra.run.smithi174.stdout:{"epoch":1,"fsid":"0ed4dd4c-ebb2-11ee-95d0-87774f69a715","modified":"2024-03-26T20:48:04.639095Z","created":"2024-03-26T20:48:04.639095Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi063","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:3300","nonce":0},{"type":"v1","addr":"172.21.15.63:6789","nonce":0}]},"addr":"172.21.15.63:6789/0","public_addr":"172.21.15.63:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-03-26T20:49:49.338 INFO:teuthology.orchestra.run.smithi174.stderr:dumped monmap epoch 1 2024-03-26T20:49:50.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:50 smithi063 ceph-mon[24818]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-26T20:49:50.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:50 smithi063 ceph-mon[24818]: from='client.? 172.21.15.174:0/2482665380' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-26T20:49:50.889 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-03-26T20:49:50.889 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -- ceph mon dump -f json 2024-03-26T20:49:52.604 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:52 smithi063 ceph-mon[24818]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-26T20:49:52.758 INFO:teuthology.orchestra.run.smithi174.stdout: 2024-03-26T20:49:52.758 INFO:teuthology.orchestra.run.smithi174.stdout:{"epoch":1,"fsid":"0ed4dd4c-ebb2-11ee-95d0-87774f69a715","modified":"2024-03-26T20:48:04.639095Z","created":"2024-03-26T20:48:04.639095Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi063","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:3300","nonce":0},{"type":"v1","addr":"172.21.15.63:6789","nonce":0}]},"addr":"172.21.15.63:6789/0","public_addr":"172.21.15.63:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-03-26T20:49:52.760 INFO:teuthology.orchestra.run.smithi174.stderr:dumped monmap epoch 1 2024-03-26T20:49:53.798 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:53 smithi063 ceph-mon[24818]: from='client.? 172.21.15.174:0/2511740169' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-26T20:49:54.259 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-03-26T20:49:54.259 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -- ceph mon dump -f json 2024-03-26T20:49:54.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:54 smithi063 ceph-mon[24818]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-26T20:49:56.144 INFO:teuthology.orchestra.run.smithi174.stdout: 2024-03-26T20:49:56.144 INFO:teuthology.orchestra.run.smithi174.stdout:{"epoch":1,"fsid":"0ed4dd4c-ebb2-11ee-95d0-87774f69a715","modified":"2024-03-26T20:48:04.639095Z","created":"2024-03-26T20:48:04.639095Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi063","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:3300","nonce":0},{"type":"v1","addr":"172.21.15.63:6789","nonce":0}]},"addr":"172.21.15.63:6789/0","public_addr":"172.21.15.63:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-03-26T20:49:56.145 INFO:teuthology.orchestra.run.smithi174.stderr:dumped monmap epoch 1 2024-03-26T20:49:56.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:56 smithi063 ceph-mon[24818]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-26T20:49:56.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:56 smithi063 ceph-mon[24818]: from='client.? 172.21.15.174:0/1609806451' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-26T20:49:57.754 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-03-26T20:49:57.754 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -- ceph mon dump -f json 2024-03-26T20:49:58.052 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:57 smithi063 ceph-mon[24818]: from='mgr.14162 172.21.15.63:0/3544862618' entity='mgr.smithi063.lxhujt' 2024-03-26T20:49:58.052 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:57 smithi063 ceph-mon[24818]: Deploying daemon node-exporter.smithi063 on smithi063 2024-03-26T20:49:58.052 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:57 smithi063 ceph-mon[24818]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-26T20:49:59.556 INFO:teuthology.orchestra.run.smithi174.stdout: 2024-03-26T20:49:59.556 INFO:teuthology.orchestra.run.smithi174.stdout:{"epoch":1,"fsid":"0ed4dd4c-ebb2-11ee-95d0-87774f69a715","modified":"2024-03-26T20:48:04.639095Z","created":"2024-03-26T20:48:04.639095Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi063","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:3300","nonce":0},{"type":"v1","addr":"172.21.15.63:6789","nonce":0}]},"addr":"172.21.15.63:6789/0","public_addr":"172.21.15.63:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-03-26T20:49:59.558 INFO:teuthology.orchestra.run.smithi174.stderr:dumped monmap epoch 1 2024-03-26T20:49:59.600 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:49:59 smithi063 ceph-mon[24818]: from='mgr.14162 172.21.15.63:0/3544862618' entity='mgr.smithi063.lxhujt' 2024-03-26T20:50:00.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:00 smithi063 ceph-mon[24818]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-26T20:50:00.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:00 smithi063 ceph-mon[24818]: from='client.? 172.21.15.174:0/3956417429' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-26T20:50:00.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:00 smithi063 ceph-mon[24818]: from='mgr.14162 172.21.15.63:0/3544862618' entity='mgr.smithi063.lxhujt' 2024-03-26T20:50:00.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:00 smithi063 ceph-mon[24818]: from='mgr.14162 172.21.15.63:0/3544862618' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-03-26T20:50:01.099 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-03-26T20:50:01.099 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -- ceph mon dump -f json 2024-03-26T20:50:02.153 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:01 smithi063 ceph-mon[24818]: from='mgr.14162 172.21.15.63:0/3544862618' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-03-26T20:50:02.153 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:01 smithi063 ceph-mon[24818]: mgrmap e14: smithi063.lxhujt(active, since 48s) 2024-03-26T20:50:02.957 INFO:teuthology.orchestra.run.smithi174.stdout: 2024-03-26T20:50:02.957 INFO:teuthology.orchestra.run.smithi174.stdout:{"epoch":1,"fsid":"0ed4dd4c-ebb2-11ee-95d0-87774f69a715","modified":"2024-03-26T20:48:04.639095Z","created":"2024-03-26T20:48:04.639095Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi063","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:3300","nonce":0},{"type":"v1","addr":"172.21.15.63:6789","nonce":0}]},"addr":"172.21.15.63:6789/0","public_addr":"172.21.15.63:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-03-26T20:50:02.959 INFO:teuthology.orchestra.run.smithi174.stderr:dumped monmap epoch 1 2024-03-26T20:50:03.551 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:03 smithi063 ceph-mon[24818]: from='client.? 172.21.15.174:0/813192735' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-26T20:50:04.510 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-03-26T20:50:04.510 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -- ceph mon dump -f json 2024-03-26T20:50:06.397 INFO:teuthology.orchestra.run.smithi174.stdout: 2024-03-26T20:50:06.397 INFO:teuthology.orchestra.run.smithi174.stdout:{"epoch":1,"fsid":"0ed4dd4c-ebb2-11ee-95d0-87774f69a715","modified":"2024-03-26T20:48:04.639095Z","created":"2024-03-26T20:48:04.639095Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi063","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:3300","nonce":0},{"type":"v1","addr":"172.21.15.63:6789","nonce":0}]},"addr":"172.21.15.63:6789/0","public_addr":"172.21.15.63:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-03-26T20:50:06.399 INFO:teuthology.orchestra.run.smithi174.stderr:dumped monmap epoch 1 2024-03-26T20:50:06.642 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:06 smithi063 ceph-mon[24818]: from='client.? 172.21.15.174:0/1744035971' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-26T20:50:07.782 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:07 smithi063 ceph-mon[24818]: Active manager daemon smithi063.lxhujt restarted 2024-03-26T20:50:07.782 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:07 smithi063 ceph-mon[24818]: Activating manager daemon smithi063.lxhujt 2024-03-26T20:50:07.782 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:07 smithi063 ceph-mon[24818]: osdmap e5: 0 total, 0 up, 0 in 2024-03-26T20:50:07.988 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-03-26T20:50:07.989 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -- ceph mon dump -f json 2024-03-26T20:50:08.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:08 smithi063 ceph-mon[24818]: mgrmap e15: smithi063.lxhujt(active, starting, since 0.543511s) 2024-03-26T20:50:08.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:08 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mon metadata", "id": "smithi063"}]: dispatch 2024-03-26T20:50:08.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:08 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mgr metadata", "who": "smithi063.lxhujt", "id": "smithi063.lxhujt"}]: dispatch 2024-03-26T20:50:08.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:08 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-26T20:50:08.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:08 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-26T20:50:08.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:08 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-26T20:50:08.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:08 smithi063 ceph-mon[24818]: Manager daemon smithi063.lxhujt is now available 2024-03-26T20:50:08.803 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:08 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T20:50:08.803 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:08 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T20:50:08.803 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:08 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:50:08.803 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:08 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T20:50:08.803 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:08 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:50:08.803 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:08 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi063.lxhujt/mirror_snapshot_schedule"}]: dispatch 2024-03-26T20:50:08.803 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:08 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi063.lxhujt/trash_purge_schedule"}]: dispatch 2024-03-26T20:50:08.803 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:08 smithi063 ceph-mon[24818]: [26/Mar/2024:20:50:07] ENGINE Bus STARTING 2024-03-26T20:50:08.803 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:08 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:50:08.803 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:08 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:50:09.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:09 smithi063 ceph-mon[24818]: [26/Mar/2024:20:50:08] ENGINE Serving on https://172.21.15.63:7150 2024-03-26T20:50:09.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:09 smithi063 ceph-mon[24818]: [26/Mar/2024:20:50:08] ENGINE Bus STARTED 2024-03-26T20:50:09.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:09 smithi063 ceph-mon[24818]: mgrmap e16: smithi063.lxhujt(active, since 1.54935s) 2024-03-26T20:50:09.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:09 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:50:09.850 INFO:teuthology.orchestra.run.smithi174.stdout: 2024-03-26T20:50:09.850 INFO:teuthology.orchestra.run.smithi174.stdout:{"epoch":1,"fsid":"0ed4dd4c-ebb2-11ee-95d0-87774f69a715","modified":"2024-03-26T20:48:04.639095Z","created":"2024-03-26T20:48:04.639095Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi063","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:3300","nonce":0},{"type":"v1","addr":"172.21.15.63:6789","nonce":0}]},"addr":"172.21.15.63:6789/0","public_addr":"172.21.15.63:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-03-26T20:50:09.851 INFO:teuthology.orchestra.run.smithi174.stderr:dumped monmap epoch 1 2024-03-26T20:50:10.741 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:10 smithi063 ceph-mon[24818]: mgrmap e17: smithi063.lxhujt(active, since 2s) 2024-03-26T20:50:10.741 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:10 smithi063 ceph-mon[24818]: from='client.? 172.21.15.174:0/1671951975' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-26T20:50:10.741 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:10 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:50:11.451 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-03-26T20:50:11.452 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -- ceph mon dump -f json 2024-03-26T20:50:12.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:11 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:50:12.052 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:11 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:50:12.052 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:11 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "who": "osd/host:smithi063", "name": "osd_memory_target"}]: dispatch 2024-03-26T20:50:12.052 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:11 smithi063 ceph-mon[24818]: Updating smithi063:/etc/ceph/ceph.conf 2024-03-26T20:50:12.052 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:11 smithi063 ceph-mon[24818]: Updating smithi063:/etc/ceph/ceph.client.admin.keyring 2024-03-26T20:50:13.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:12 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:50:14.051 INFO:teuthology.orchestra.run.smithi174.stdout: 2024-03-26T20:50:14.051 INFO:teuthology.orchestra.run.smithi174.stdout:{"epoch":1,"fsid":"0ed4dd4c-ebb2-11ee-95d0-87774f69a715","modified":"2024-03-26T20:48:04.639095Z","created":"2024-03-26T20:48:04.639095Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi063","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:3300","nonce":0},{"type":"v1","addr":"172.21.15.63:6789","nonce":0}]},"addr":"172.21.15.63:6789/0","public_addr":"172.21.15.63:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-03-26T20:50:14.053 INFO:teuthology.orchestra.run.smithi174.stderr:dumped monmap epoch 1 2024-03-26T20:50:14.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:14 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:50:14.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:14 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:50:14.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:14 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "who": "osd/host:smithi174", "name": "osd_memory_target"}]: dispatch 2024-03-26T20:50:14.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:14 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:50:14.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:14 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi174", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-03-26T20:50:14.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:14 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi174", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-03-26T20:50:14.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:14 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:50:14.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:14 smithi063 ceph-mon[24818]: Deploying daemon crash.smithi174 on smithi174 2024-03-26T20:50:15.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:15 smithi063 ceph-mon[24818]: from='client.? 172.21.15.174:0/1219409134' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-26T20:50:16.642 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-03-26T20:50:16.643 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -- ceph mon dump -f json 2024-03-26T20:50:17.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:17 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:50:17.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:17 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi174.sexoin", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-26T20:50:17.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:17 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.smithi174.sexoin", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-03-26T20:50:17.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:17 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-26T20:50:17.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:17 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:50:17.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:17 smithi063 ceph-mon[24818]: Deploying daemon mgr.smithi174.sexoin on smithi174 2024-03-26T20:50:18.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:18 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:50:19.601 INFO:teuthology.orchestra.run.smithi174.stdout: 2024-03-26T20:50:19.602 INFO:teuthology.orchestra.run.smithi174.stdout:{"epoch":1,"fsid":"0ed4dd4c-ebb2-11ee-95d0-87774f69a715","modified":"2024-03-26T20:48:04.639095Z","created":"2024-03-26T20:48:04.639095Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi063","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:3300","nonce":0},{"type":"v1","addr":"172.21.15.63:6789","nonce":0}]},"addr":"172.21.15.63:6789/0","public_addr":"172.21.15.63:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-03-26T20:50:19.604 INFO:teuthology.orchestra.run.smithi174.stderr:dumped monmap epoch 1 2024-03-26T20:50:20.052 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:19 smithi063 ceph-mon[24818]: from='client.? 172.21.15.174:0/1881090301' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-26T20:50:21.304 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-03-26T20:50:21.304 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -- ceph mon dump -f json 2024-03-26T20:50:22.052 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:21 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:50:22.052 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:21 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-26T20:50:22.052 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:21 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:50:22.052 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:21 smithi063 ceph-mon[24818]: Deploying daemon mon.smithi174 on smithi174 2024-03-26T20:50:23.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:23 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:50:24.734 INFO:teuthology.orchestra.run.smithi174.stdout: 2024-03-26T20:50:24.734 INFO:teuthology.orchestra.run.smithi174.stdout:{"epoch":1,"fsid":"0ed4dd4c-ebb2-11ee-95d0-87774f69a715","modified":"2024-03-26T20:48:04.639095Z","created":"2024-03-26T20:48:04.639095Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi063","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:3300","nonce":0},{"type":"v1","addr":"172.21.15.63:6789","nonce":0}]},"addr":"172.21.15.63:6789/0","public_addr":"172.21.15.63:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-03-26T20:50:24.735 INFO:teuthology.orchestra.run.smithi174.stderr:dumped monmap epoch 1 2024-03-26T20:50:25.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:24 smithi063 ceph-mon[24818]: from='client.? 172.21.15.174:0/330145611' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-26T20:50:26.412 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-03-26T20:50:26.413 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -- ceph mon dump -f json 2024-03-26T20:50:30.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:29 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mon metadata", "id": "smithi063"}]: dispatch 2024-03-26T20:50:30.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:29 smithi063 ceph-mon[24818]: mon.smithi063 calling monitor election 2024-03-26T20:50:30.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:29 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mon metadata", "id": "smithi174"}]: dispatch 2024-03-26T20:50:30.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:29 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mon metadata", "id": "smithi174"}]: dispatch 2024-03-26T20:50:30.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:29 smithi063 ceph-mon[24818]: from='mgr.? 172.21.15.174:0/1104399308' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi174.sexoin/crt"}]: dispatch 2024-03-26T20:50:30.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:29 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mon metadata", "id": "smithi174"}]: dispatch 2024-03-26T20:50:30.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:29 smithi063 ceph-mon[24818]: mon.smithi174 calling monitor election 2024-03-26T20:50:30.303 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:29 smithi063 ceph-mon[24818]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-26T20:50:30.303 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:29 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mon metadata", "id": "smithi174"}]: dispatch 2024-03-26T20:50:30.303 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:29 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mon metadata", "id": "smithi174"}]: dispatch 2024-03-26T20:50:30.303 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:29 smithi063 ceph-mon[24818]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-26T20:50:30.303 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:29 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mon metadata", "id": "smithi174"}]: dispatch 2024-03-26T20:50:30.303 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:29 smithi063 ceph-mon[24818]: mon.smithi063 is new leader, mons smithi063,smithi174 in quorum (ranks 0,1) 2024-03-26T20:50:30.303 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:29 smithi063 ceph-mon[24818]: monmap e2: 2 mons at {smithi063=[v2:172.21.15.63:3300/0,v1:172.21.15.63:6789/0],smithi174=[v2:172.21.15.174:3300/0,v1:172.21.15.174:6789/0]} 2024-03-26T20:50:30.303 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:29 smithi063 ceph-mon[24818]: fsmap 2024-03-26T20:50:30.303 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:29 smithi063 ceph-mon[24818]: osdmap e5: 0 total, 0 up, 0 in 2024-03-26T20:50:30.303 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:29 smithi063 ceph-mon[24818]: mgrmap e17: smithi063.lxhujt(active, since 22s) 2024-03-26T20:50:30.303 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:29 smithi063 ceph-mon[24818]: Standby manager daemon smithi174.sexoin started 2024-03-26T20:50:30.303 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:29 smithi063 ceph-mon[24818]: from='mgr.? 172.21.15.174:0/1104399308' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-26T20:50:30.303 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:29 smithi063 ceph-mon[24818]: overall HEALTH_OK 2024-03-26T20:50:30.303 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:29 smithi063 ceph-mon[24818]: from='mgr.? 172.21.15.174:0/1104399308' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi174.sexoin/key"}]: dispatch 2024-03-26T20:50:30.304 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:29 smithi063 ceph-mon[24818]: from='mgr.? 172.21.15.174:0/1104399308' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-26T20:50:30.304 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:29 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:50:30.304 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:29 smithi063 ceph-mon[24818]: Deploying daemon node-exporter.smithi174 on smithi174 2024-03-26T20:50:31.203 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:50:30 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mon metadata", "id": "smithi174"}]: dispatch 2024-03-26T20:50:31.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:30 smithi063 ceph-mon[24818]: mgrmap e18: smithi063.lxhujt(active, since 23s), standbys: smithi174.sexoin 2024-03-26T20:50:31.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:30 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mgr metadata", "who": "smithi174.sexoin", "id": "smithi174.sexoin"}]: dispatch 2024-03-26T20:50:31.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:30 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mon metadata", "id": "smithi174"}]: dispatch 2024-03-26T20:50:31.306 INFO:teuthology.orchestra.run.smithi174.stdout: 2024-03-26T20:50:31.306 INFO:teuthology.orchestra.run.smithi174.stdout:{"epoch":2,"fsid":"0ed4dd4c-ebb2-11ee-95d0-87774f69a715","modified":"2024-03-26T20:50:24.841910Z","created":"2024-03-26T20:48:04.639095Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi063","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:3300","nonce":0},{"type":"v1","addr":"172.21.15.63:6789","nonce":0}]},"addr":"172.21.15.63:6789/0","public_addr":"172.21.15.63:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"smithi174","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:3300","nonce":0},{"type":"v1","addr":"172.21.15.174:6789","nonce":0}]},"addr":"172.21.15.174:6789/0","public_addr":"172.21.15.174:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-03-26T20:50:31.307 INFO:teuthology.orchestra.run.smithi174.stderr:dumped monmap epoch 2 2024-03-26T20:50:32.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:50:31 smithi174 ceph-mon[26934]: from='client.? 172.21.15.174:0/490411353' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-26T20:50:32.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:50:31 smithi174 ceph-mon[26934]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-26T20:50:32.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:31 smithi063 ceph-mon[24818]: from='client.? 172.21.15.174:0/490411353' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-26T20:50:32.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:31 smithi063 ceph-mon[24818]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-26T20:50:32.879 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-03-26T20:50:32.880 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -- ceph config generate-minimal-conf 2024-03-26T20:50:33.515 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:50:33 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:50:33.719 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:33 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:50:34.819 INFO:teuthology.orchestra.run.smithi063.stdout:# minimal ceph.conf for 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 2024-03-26T20:50:34.819 INFO:teuthology.orchestra.run.smithi063.stdout:[global] 2024-03-26T20:50:34.819 INFO:teuthology.orchestra.run.smithi063.stdout: fsid = 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 2024-03-26T20:50:34.819 INFO:teuthology.orchestra.run.smithi063.stdout: mon_host = [v2:172.21.15.63:3300/0,v1:172.21.15.63:6789/0] [v2:172.21.15.174:3300/0,v1:172.21.15.174:6789/0] 2024-03-26T20:50:35.047 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:34 smithi063 ceph-mon[24818]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-26T20:50:35.047 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:34 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:50:35.047 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:34 smithi063 ceph-mon[24818]: Deploying daemon prometheus.smithi063 on smithi063 2024-03-26T20:50:35.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:50:34 smithi174 ceph-mon[26934]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-26T20:50:35.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:50:34 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:50:35.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:50:34 smithi174 ceph-mon[26934]: Deploying daemon prometheus.smithi063 on smithi063 2024-03-26T20:50:35.418 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-03-26T20:50:35.418 DEBUG:teuthology.orchestra.run.smithi063:> set -ex 2024-03-26T20:50:35.418 DEBUG:teuthology.orchestra.run.smithi063:> sudo dd of=/etc/ceph/ceph.conf 2024-03-26T20:50:35.450 DEBUG:teuthology.orchestra.run.smithi063:> set -ex 2024-03-26T20:50:35.450 DEBUG:teuthology.orchestra.run.smithi063:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-03-26T20:50:35.517 DEBUG:teuthology.orchestra.run.smithi174:> set -ex 2024-03-26T20:50:35.517 DEBUG:teuthology.orchestra.run.smithi174:> sudo dd of=/etc/ceph/ceph.conf 2024-03-26T20:50:35.547 DEBUG:teuthology.orchestra.run.smithi174:> set -ex 2024-03-26T20:50:35.548 DEBUG:teuthology.orchestra.run.smithi174:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-03-26T20:50:35.615 INFO:tasks.cephadm:Deploying OSDs... 2024-03-26T20:50:35.615 DEBUG:teuthology.orchestra.run.smithi063:> set -ex 2024-03-26T20:50:35.615 DEBUG:teuthology.orchestra.run.smithi063:> dd if=/scratch_devs of=/dev/stdout 2024-03-26T20:50:35.630 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-03-26T20:50:35.630 DEBUG:teuthology.orchestra.run.smithi063:> stat /dev/vg_nvme/lv_1 2024-03-26T20:50:35.685 INFO:teuthology.orchestra.run.smithi063.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-03-26T20:50:35.685 INFO:teuthology.orchestra.run.smithi063.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-26T20:50:35.685 INFO:teuthology.orchestra.run.smithi063.stdout:Device: 5h/5d Inode: 667 Links: 1 2024-03-26T20:50:35.685 INFO:teuthology.orchestra.run.smithi063.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-26T20:50:35.685 INFO:teuthology.orchestra.run.smithi063.stdout:Context: system_u:object_r:device_t:s0 2024-03-26T20:50:35.685 INFO:teuthology.orchestra.run.smithi063.stdout:Access: 2024-03-26 20:49:57.449871976 +0000 2024-03-26T20:50:35.685 INFO:teuthology.orchestra.run.smithi063.stdout:Modify: 2024-03-26 20:47:02.499322546 +0000 2024-03-26T20:50:35.686 INFO:teuthology.orchestra.run.smithi063.stdout:Change: 2024-03-26 20:47:02.499322546 +0000 2024-03-26T20:50:35.686 INFO:teuthology.orchestra.run.smithi063.stdout: Birth: 2024-03-26 20:47:02.499322546 +0000 2024-03-26T20:50:35.686 DEBUG:teuthology.orchestra.run.smithi063:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-03-26T20:50:35.751 INFO:teuthology.orchestra.run.smithi063.stderr:1+0 records in 2024-03-26T20:50:35.751 INFO:teuthology.orchestra.run.smithi063.stderr:1+0 records out 2024-03-26T20:50:35.751 INFO:teuthology.orchestra.run.smithi063.stderr:512 bytes copied, 0.000247707 s, 2.1 MB/s 2024-03-26T20:50:35.752 DEBUG:teuthology.orchestra.run.smithi063:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-03-26T20:50:35.807 DEBUG:teuthology.orchestra.run.smithi063:> stat /dev/vg_nvme/lv_2 2024-03-26T20:50:35.861 INFO:teuthology.orchestra.run.smithi063.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-03-26T20:50:35.861 INFO:teuthology.orchestra.run.smithi063.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-26T20:50:35.861 INFO:teuthology.orchestra.run.smithi063.stdout:Device: 5h/5d Inode: 684 Links: 1 2024-03-26T20:50:35.862 INFO:teuthology.orchestra.run.smithi063.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-26T20:50:35.862 INFO:teuthology.orchestra.run.smithi063.stdout:Context: system_u:object_r:device_t:s0 2024-03-26T20:50:35.862 INFO:teuthology.orchestra.run.smithi063.stdout:Access: 2024-03-26 20:49:57.450871967 +0000 2024-03-26T20:50:35.862 INFO:teuthology.orchestra.run.smithi063.stdout:Modify: 2024-03-26 20:47:02.838319511 +0000 2024-03-26T20:50:35.862 INFO:teuthology.orchestra.run.smithi063.stdout:Change: 2024-03-26 20:47:02.838319511 +0000 2024-03-26T20:50:35.862 INFO:teuthology.orchestra.run.smithi063.stdout: Birth: 2024-03-26 20:47:02.838319511 +0000 2024-03-26T20:50:35.862 DEBUG:teuthology.orchestra.run.smithi063:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-03-26T20:50:35.927 INFO:teuthology.orchestra.run.smithi063.stderr:1+0 records in 2024-03-26T20:50:35.927 INFO:teuthology.orchestra.run.smithi063.stderr:1+0 records out 2024-03-26T20:50:35.927 INFO:teuthology.orchestra.run.smithi063.stderr:512 bytes copied, 0.000212029 s, 2.4 MB/s 2024-03-26T20:50:35.928 DEBUG:teuthology.orchestra.run.smithi063:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-03-26T20:50:35.984 DEBUG:teuthology.orchestra.run.smithi063:> stat /dev/vg_nvme/lv_3 2024-03-26T20:50:36.038 INFO:teuthology.orchestra.run.smithi063.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-03-26T20:50:36.038 INFO:teuthology.orchestra.run.smithi063.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-26T20:50:36.038 INFO:teuthology.orchestra.run.smithi063.stdout:Device: 5h/5d Inode: 706 Links: 1 2024-03-26T20:50:36.038 INFO:teuthology.orchestra.run.smithi063.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-26T20:50:36.038 INFO:teuthology.orchestra.run.smithi063.stdout:Context: system_u:object_r:device_t:s0 2024-03-26T20:50:36.038 INFO:teuthology.orchestra.run.smithi063.stdout:Access: 2024-03-26 20:49:57.450871967 +0000 2024-03-26T20:50:36.039 INFO:teuthology.orchestra.run.smithi063.stdout:Modify: 2024-03-26 20:47:03.175316495 +0000 2024-03-26T20:50:36.039 INFO:teuthology.orchestra.run.smithi063.stdout:Change: 2024-03-26 20:47:03.175316495 +0000 2024-03-26T20:50:36.039 INFO:teuthology.orchestra.run.smithi063.stdout: Birth: 2024-03-26 20:47:03.175316495 +0000 2024-03-26T20:50:36.039 DEBUG:teuthology.orchestra.run.smithi063:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-03-26T20:50:36.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:35 smithi063 ceph-mon[24818]: from='client.? 172.21.15.63:0/2005228036' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:50:36.077 INFO:teuthology.orchestra.run.smithi063.stderr:1+0 records in 2024-03-26T20:50:36.077 INFO:teuthology.orchestra.run.smithi063.stderr:1+0 records out 2024-03-26T20:50:36.077 INFO:teuthology.orchestra.run.smithi063.stderr:512 bytes copied, 0.000174987 s, 2.9 MB/s 2024-03-26T20:50:36.078 DEBUG:teuthology.orchestra.run.smithi063:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-03-26T20:50:36.133 DEBUG:teuthology.orchestra.run.smithi063:> stat /dev/vg_nvme/lv_4 2024-03-26T20:50:36.189 INFO:teuthology.orchestra.run.smithi063.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-03-26T20:50:36.189 INFO:teuthology.orchestra.run.smithi063.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-26T20:50:36.189 INFO:teuthology.orchestra.run.smithi063.stdout:Device: 5h/5d Inode: 722 Links: 1 2024-03-26T20:50:36.189 INFO:teuthology.orchestra.run.smithi063.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-26T20:50:36.189 INFO:teuthology.orchestra.run.smithi063.stdout:Context: system_u:object_r:device_t:s0 2024-03-26T20:50:36.190 INFO:teuthology.orchestra.run.smithi063.stdout:Access: 2024-03-26 20:49:57.450871967 +0000 2024-03-26T20:50:36.190 INFO:teuthology.orchestra.run.smithi063.stdout:Modify: 2024-03-26 20:47:03.531313308 +0000 2024-03-26T20:50:36.190 INFO:teuthology.orchestra.run.smithi063.stdout:Change: 2024-03-26 20:47:03.531313308 +0000 2024-03-26T20:50:36.190 INFO:teuthology.orchestra.run.smithi063.stdout: Birth: 2024-03-26 20:47:03.531313308 +0000 2024-03-26T20:50:36.190 DEBUG:teuthology.orchestra.run.smithi063:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-03-26T20:50:36.257 INFO:teuthology.orchestra.run.smithi063.stderr:1+0 records in 2024-03-26T20:50:36.257 INFO:teuthology.orchestra.run.smithi063.stderr:1+0 records out 2024-03-26T20:50:36.257 INFO:teuthology.orchestra.run.smithi063.stderr:512 bytes copied, 0.000282271 s, 1.8 MB/s 2024-03-26T20:50:36.258 DEBUG:teuthology.orchestra.run.smithi063:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-03-26T20:50:36.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:50:35 smithi174 ceph-mon[26934]: from='client.? 172.21.15.63:0/2005228036' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:50:36.315 DEBUG:teuthology.orchestra.run.smithi174:> set -ex 2024-03-26T20:50:36.315 DEBUG:teuthology.orchestra.run.smithi174:> dd if=/scratch_devs of=/dev/stdout 2024-03-26T20:50:36.333 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-03-26T20:50:36.333 DEBUG:teuthology.orchestra.run.smithi174:> stat /dev/vg_nvme/lv_1 2024-03-26T20:50:36.390 INFO:teuthology.orchestra.run.smithi174.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-03-26T20:50:36.390 INFO:teuthology.orchestra.run.smithi174.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-26T20:50:36.390 INFO:teuthology.orchestra.run.smithi174.stdout:Device: 5h/5d Inode: 661 Links: 1 2024-03-26T20:50:36.391 INFO:teuthology.orchestra.run.smithi174.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-26T20:50:36.391 INFO:teuthology.orchestra.run.smithi174.stdout:Context: system_u:object_r:device_t:s0 2024-03-26T20:50:36.391 INFO:teuthology.orchestra.run.smithi174.stdout:Access: 2024-03-26 20:50:30.483855357 +0000 2024-03-26T20:50:36.391 INFO:teuthology.orchestra.run.smithi174.stdout:Modify: 2024-03-26 20:46:58.363526452 +0000 2024-03-26T20:50:36.391 INFO:teuthology.orchestra.run.smithi174.stdout:Change: 2024-03-26 20:46:58.363526452 +0000 2024-03-26T20:50:36.391 INFO:teuthology.orchestra.run.smithi174.stdout: Birth: 2024-03-26 20:46:58.363526452 +0000 2024-03-26T20:50:36.391 DEBUG:teuthology.orchestra.run.smithi174:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-03-26T20:50:36.460 INFO:teuthology.orchestra.run.smithi174.stderr:1+0 records in 2024-03-26T20:50:36.460 INFO:teuthology.orchestra.run.smithi174.stderr:1+0 records out 2024-03-26T20:50:36.460 INFO:teuthology.orchestra.run.smithi174.stderr:512 bytes copied, 0.000238058 s, 2.2 MB/s 2024-03-26T20:50:36.462 DEBUG:teuthology.orchestra.run.smithi174:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-03-26T20:50:36.521 DEBUG:teuthology.orchestra.run.smithi174:> stat /dev/vg_nvme/lv_2 2024-03-26T20:50:36.578 INFO:teuthology.orchestra.run.smithi174.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-03-26T20:50:36.578 INFO:teuthology.orchestra.run.smithi174.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-26T20:50:36.578 INFO:teuthology.orchestra.run.smithi174.stdout:Device: 5h/5d Inode: 680 Links: 1 2024-03-26T20:50:36.579 INFO:teuthology.orchestra.run.smithi174.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-26T20:50:36.579 INFO:teuthology.orchestra.run.smithi174.stdout:Context: system_u:object_r:device_t:s0 2024-03-26T20:50:36.579 INFO:teuthology.orchestra.run.smithi174.stdout:Access: 2024-03-26 20:50:30.484855344 +0000 2024-03-26T20:50:36.579 INFO:teuthology.orchestra.run.smithi174.stdout:Modify: 2024-03-26 20:46:58.716522074 +0000 2024-03-26T20:50:36.579 INFO:teuthology.orchestra.run.smithi174.stdout:Change: 2024-03-26 20:46:58.716522074 +0000 2024-03-26T20:50:36.579 INFO:teuthology.orchestra.run.smithi174.stdout: Birth: 2024-03-26 20:46:58.716522074 +0000 2024-03-26T20:50:36.579 DEBUG:teuthology.orchestra.run.smithi174:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-03-26T20:50:36.646 INFO:teuthology.orchestra.run.smithi174.stderr:1+0 records in 2024-03-26T20:50:36.646 INFO:teuthology.orchestra.run.smithi174.stderr:1+0 records out 2024-03-26T20:50:36.646 INFO:teuthology.orchestra.run.smithi174.stderr:512 bytes copied, 0.000196178 s, 2.6 MB/s 2024-03-26T20:50:36.647 DEBUG:teuthology.orchestra.run.smithi174:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-03-26T20:50:36.704 DEBUG:teuthology.orchestra.run.smithi174:> stat /dev/vg_nvme/lv_3 2024-03-26T20:50:36.760 INFO:teuthology.orchestra.run.smithi174.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-03-26T20:50:36.760 INFO:teuthology.orchestra.run.smithi174.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-26T20:50:36.760 INFO:teuthology.orchestra.run.smithi174.stdout:Device: 5h/5d Inode: 700 Links: 1 2024-03-26T20:50:36.760 INFO:teuthology.orchestra.run.smithi174.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-26T20:50:36.760 INFO:teuthology.orchestra.run.smithi174.stdout:Context: system_u:object_r:device_t:s0 2024-03-26T20:50:36.760 INFO:teuthology.orchestra.run.smithi174.stdout:Access: 2024-03-26 20:50:30.484855344 +0000 2024-03-26T20:50:36.760 INFO:teuthology.orchestra.run.smithi174.stdout:Modify: 2024-03-26 20:46:59.057517844 +0000 2024-03-26T20:50:36.761 INFO:teuthology.orchestra.run.smithi174.stdout:Change: 2024-03-26 20:46:59.057517844 +0000 2024-03-26T20:50:36.761 INFO:teuthology.orchestra.run.smithi174.stdout: Birth: 2024-03-26 20:46:59.057517844 +0000 2024-03-26T20:50:36.761 DEBUG:teuthology.orchestra.run.smithi174:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-03-26T20:50:36.828 INFO:teuthology.orchestra.run.smithi174.stderr:1+0 records in 2024-03-26T20:50:36.828 INFO:teuthology.orchestra.run.smithi174.stderr:1+0 records out 2024-03-26T20:50:36.828 INFO:teuthology.orchestra.run.smithi174.stderr:512 bytes copied, 0.000277837 s, 1.8 MB/s 2024-03-26T20:50:36.829 DEBUG:teuthology.orchestra.run.smithi174:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-03-26T20:50:36.886 DEBUG:teuthology.orchestra.run.smithi174:> stat /dev/vg_nvme/lv_4 2024-03-26T20:50:36.942 INFO:teuthology.orchestra.run.smithi174.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-03-26T20:50:36.942 INFO:teuthology.orchestra.run.smithi174.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-26T20:50:36.942 INFO:teuthology.orchestra.run.smithi174.stdout:Device: 5h/5d Inode: 719 Links: 1 2024-03-26T20:50:36.942 INFO:teuthology.orchestra.run.smithi174.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-26T20:50:36.942 INFO:teuthology.orchestra.run.smithi174.stdout:Context: system_u:object_r:device_t:s0 2024-03-26T20:50:36.942 INFO:teuthology.orchestra.run.smithi174.stdout:Access: 2024-03-26 20:50:30.485855332 +0000 2024-03-26T20:50:36.942 INFO:teuthology.orchestra.run.smithi174.stdout:Modify: 2024-03-26 20:46:59.395513652 +0000 2024-03-26T20:50:36.943 INFO:teuthology.orchestra.run.smithi174.stdout:Change: 2024-03-26 20:46:59.395513652 +0000 2024-03-26T20:50:36.943 INFO:teuthology.orchestra.run.smithi174.stdout: Birth: 2024-03-26 20:46:59.395513652 +0000 2024-03-26T20:50:36.943 DEBUG:teuthology.orchestra.run.smithi174:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-03-26T20:50:37.008 INFO:teuthology.orchestra.run.smithi174.stderr:1+0 records in 2024-03-26T20:50:37.008 INFO:teuthology.orchestra.run.smithi174.stderr:1+0 records out 2024-03-26T20:50:37.008 INFO:teuthology.orchestra.run.smithi174.stderr:512 bytes copied, 0.000201839 s, 2.5 MB/s 2024-03-26T20:50:37.009 DEBUG:teuthology.orchestra.run.smithi174:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-03-26T20:50:37.065 INFO:tasks.cephadm:Deploying osd.0 on smithi063 with /dev/vg_nvme/lv_4... 2024-03-26T20:50:37.065 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-03-26T20:50:37.090 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:36 smithi063 ceph-mon[24818]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-26T20:50:37.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:50:36 smithi174 ceph-mon[26934]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-26T20:50:38.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:50:38 smithi174 ceph-mon[26934]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-26T20:50:38.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:50:38 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:50:38.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:38 smithi063 ceph-mon[24818]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-26T20:50:38.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:38 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:50:40.783 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:40 smithi063 ceph-mon[24818]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-26T20:50:40.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:50:40 smithi174 ceph-mon[26934]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-26T20:50:41.356 INFO:teuthology.orchestra.run.smithi063.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-03-26T20:50:41.356 INFO:teuthology.orchestra.run.smithi063.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-03-26T20:50:41.356 INFO:teuthology.orchestra.run.smithi063.stderr: stderr: 10+0 records in 2024-03-26T20:50:41.356 INFO:teuthology.orchestra.run.smithi063.stderr:10+0 records out 2024-03-26T20:50:41.356 INFO:teuthology.orchestra.run.smithi063.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.011369 s, 922 MB/s 2024-03-26T20:50:41.356 INFO:teuthology.orchestra.run.smithi063.stderr:--> Zapping successful for: 2024-03-26T20:50:42.615 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -- ceph orch daemon add osd smithi063:vg_nvme/lv_4 2024-03-26T20:50:42.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:50:42 smithi174 ceph-mon[26934]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-26T20:50:42.800 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:42 smithi063 ceph-mon[24818]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-26T20:50:44.629 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:44 smithi063 ceph-mon[24818]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-26T20:50:44.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:50:44 smithi174 ceph-mon[26934]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-26T20:50:46.579 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:46 smithi063 ceph-mon[24818]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-26T20:50:46.580 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:46 smithi063 ceph-mon[24818]: from='client.14236 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi063:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:50:46.580 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:46 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-26T20:50:46.580 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:46 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-26T20:50:46.580 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:46 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:50:46.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:50:46 smithi174 ceph-mon[26934]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-26T20:50:46.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:50:46 smithi174 ceph-mon[26934]: from='client.14236 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi063:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:50:46.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:50:46 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-26T20:50:46.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:50:46 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-26T20:50:46.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:50:46 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:50:48.547 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:48 smithi063 ceph-mon[24818]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-26T20:50:48.547 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:48 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:50:48.547 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:48 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T20:50:48.547 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:48 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:50:48.547 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:48 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T20:50:48.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:50:48 smithi174 ceph-mon[26934]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-26T20:50:48.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:50:48 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:50:48.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:50:48 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T20:50:48.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:50:48 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:50:48.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:50:48 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T20:50:49.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:50:49 smithi174 ceph-mon[26934]: from='client.? 172.21.15.63:0/554860767' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6b0f79e5-c540-4701-a03e-38b2596b333b"}]: dispatch 2024-03-26T20:50:49.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:50:49 smithi174 ceph-mon[26934]: from='client.? 172.21.15.63:0/554860767' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6b0f79e5-c540-4701-a03e-38b2596b333b"}]': finished 2024-03-26T20:50:49.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:50:49 smithi174 ceph-mon[26934]: osdmap e6: 1 total, 0 up, 1 in 2024-03-26T20:50:49.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:50:49 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-26T20:50:49.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:49 smithi063 ceph-mon[24818]: from='client.? 172.21.15.63:0/554860767' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6b0f79e5-c540-4701-a03e-38b2596b333b"}]: dispatch 2024-03-26T20:50:49.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:49 smithi063 ceph-mon[24818]: from='client.? 172.21.15.63:0/554860767' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6b0f79e5-c540-4701-a03e-38b2596b333b"}]': finished 2024-03-26T20:50:49.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:49 smithi063 ceph-mon[24818]: osdmap e6: 1 total, 0 up, 1 in 2024-03-26T20:50:49.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:49 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-26T20:50:50.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:50:50 smithi174 ceph-mon[26934]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-26T20:50:50.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:50:50 smithi174 ceph-mon[26934]: from='client.? 172.21.15.63:0/3048654825' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-26T20:50:50.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:50:50 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:50:50.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:50:50 smithi174 ceph-mon[26934]: Updating smithi174:/etc/ceph/ceph.conf 2024-03-26T20:50:50.551 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:50 smithi063 ceph-mon[24818]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-26T20:50:50.551 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:50 smithi063 ceph-mon[24818]: from='client.? 172.21.15.63:0/3048654825' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-26T20:50:50.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:50 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:50:50.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:50 smithi063 ceph-mon[24818]: Updating smithi174:/etc/ceph/ceph.conf 2024-03-26T20:50:51.551 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:51 smithi063 ceph-mon[24818]: Updating smithi174:/etc/ceph/ceph.client.admin.keyring 2024-03-26T20:50:51.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:51 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:50:51.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:50:51 smithi174 ceph-mon[26934]: Updating smithi174:/etc/ceph/ceph.client.admin.keyring 2024-03-26T20:50:51.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:50:51 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:50:52.501 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:52 smithi063 ceph-mon[24818]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-26T20:50:52.501 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:52 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:50:52.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:50:52 smithi174 ceph-mon[26934]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-26T20:50:52.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:50:52 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:50:53.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:50:53 smithi174 ceph-mon[26934]: Updating smithi063:/etc/ceph/ceph.conf 2024-03-26T20:50:53.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:50:53 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:50:53.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:50:53 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:50:53.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:50:53 smithi174 ceph-mon[26934]: Reconfiguring mon.smithi063 (unknown last config time)... 2024-03-26T20:50:53.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:50:53 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-26T20:50:53.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:50:53 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-26T20:50:53.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:50:53 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:50:53.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:50:53 smithi174 ceph-mon[26934]: Reconfiguring daemon mon.smithi063 on smithi063 2024-03-26T20:50:53.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:50:53 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:50:53.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:53 smithi063 ceph-mon[24818]: Updating smithi063:/etc/ceph/ceph.conf 2024-03-26T20:50:53.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:53 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:50:53.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:53 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:50:53.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:53 smithi063 ceph-mon[24818]: Reconfiguring mon.smithi063 (unknown last config time)... 2024-03-26T20:50:53.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:53 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-26T20:50:53.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:53 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-26T20:50:53.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:53 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:50:53.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:53 smithi063 ceph-mon[24818]: Reconfiguring daemon mon.smithi063 on smithi063 2024-03-26T20:50:53.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:53 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:50:54.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:54 smithi063 ceph-mon[24818]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-26T20:50:54.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:54 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:50:54.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:54 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi063.lxhujt", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-26T20:50:54.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:54 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-26T20:50:54.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:54 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:50:54.719 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:50:54 smithi174 ceph-mon[26934]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-26T20:50:54.719 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:50:54 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:50:54.719 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:50:54 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi063.lxhujt", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-26T20:50:54.719 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:50:54 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-26T20:50:54.719 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:50:54 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:50:55.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:50:55 smithi174 ceph-mon[26934]: Reconfiguring mgr.smithi063.lxhujt (unknown last config time)... 2024-03-26T20:50:55.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:50:55 smithi174 ceph-mon[26934]: Reconfiguring daemon mgr.smithi063.lxhujt on smithi063 2024-03-26T20:50:55.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:55 smithi063 ceph-mon[24818]: Reconfiguring mgr.smithi063.lxhujt (unknown last config time)... 2024-03-26T20:50:55.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:55 smithi063 ceph-mon[24818]: Reconfiguring daemon mgr.smithi063.lxhujt on smithi063 2024-03-26T20:50:56.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:50:56 smithi174 ceph-mon[26934]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-26T20:50:56.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:50:56 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:50:56.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:56 smithi063 ceph-mon[24818]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-26T20:50:56.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:56 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:50:57.670 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:57 smithi063 ceph-mon[24818]: Reconfiguring alertmanager.smithi063 (dependencies changed)... 2024-03-26T20:50:57.670 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:57 smithi063 ceph-mon[24818]: Reconfiguring daemon alertmanager.smithi063 on smithi063 2024-03-26T20:50:57.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:50:57 smithi174 ceph-mon[26934]: Reconfiguring alertmanager.smithi063 (dependencies changed)... 2024-03-26T20:50:57.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:50:57 smithi174 ceph-mon[26934]: Reconfiguring daemon alertmanager.smithi063 on smithi063 2024-03-26T20:50:58.784 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:58 smithi063 ceph-mon[24818]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-26T20:50:58.784 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:58 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-03-26T20:50:58.784 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:58 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:50:58.784 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:50:58 smithi063 ceph-mon[24818]: Deploying daemon osd.0 on smithi063 2024-03-26T20:50:58.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:50:58 smithi174 ceph-mon[26934]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-26T20:50:58.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:50:58 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-03-26T20:50:58.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:50:58 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:50:58.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:50:58 smithi174 ceph-mon[26934]: Deploying daemon osd.0 on smithi063 2024-03-26T20:51:00.537 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:00 smithi063 ceph-mon[24818]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-26T20:51:00.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:00 smithi174 ceph-mon[26934]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-26T20:51:02.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:02 smithi174 ceph-mon[26934]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-26T20:51:02.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:02 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:51:02.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:02 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi063", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-03-26T20:51:02.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:02 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:51:02.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:02 smithi063 ceph-mon[24818]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-26T20:51:02.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:02 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:51:02.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:02 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi063", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-03-26T20:51:02.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:02 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:51:03.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:03 smithi174 ceph-mon[26934]: Reconfiguring crash.smithi063 (monmap changed)... 2024-03-26T20:51:03.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:03 smithi174 ceph-mon[26934]: Reconfiguring daemon crash.smithi063 on smithi063 2024-03-26T20:51:03.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:03 smithi063 ceph-mon[24818]: Reconfiguring crash.smithi063 (monmap changed)... 2024-03-26T20:51:03.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:03 smithi063 ceph-mon[24818]: Reconfiguring daemon crash.smithi063 on smithi063 2024-03-26T20:51:04.551 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:04 smithi063 ceph-mon[24818]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-26T20:51:04.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:04 smithi174 ceph-mon[26934]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-26T20:51:06.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:06 smithi063 ceph-mon[24818]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-26T20:51:06.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:06 smithi174 ceph-mon[26934]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-26T20:51:08.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:08 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:51:08.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:08 smithi174 ceph-mon[26934]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-26T20:51:08.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:08 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi063.lxhujt/mirror_snapshot_schedule"}]: dispatch 2024-03-26T20:51:08.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:08 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi063.lxhujt/trash_purge_schedule"}]: dispatch 2024-03-26T20:51:08.356 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:08 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:51:08.356 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:08 smithi063 ceph-mon[24818]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-26T20:51:08.356 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:08 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi063.lxhujt/mirror_snapshot_schedule"}]: dispatch 2024-03-26T20:51:08.356 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:08 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi063.lxhujt/trash_purge_schedule"}]: dispatch 2024-03-26T20:51:09.551 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:09 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:51:09.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:09 smithi063 ceph-mon[24818]: Reconfiguring grafana.smithi063 (dependencies changed)... 2024-03-26T20:51:09.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:09 smithi063 ceph-mon[24818]: Reconfiguring daemon grafana.smithi063 on smithi063 2024-03-26T20:51:09.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:09 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:51:09.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:09 smithi174 ceph-mon[26934]: Reconfiguring grafana.smithi063 (dependencies changed)... 2024-03-26T20:51:09.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:09 smithi174 ceph-mon[26934]: Reconfiguring daemon grafana.smithi063 on smithi063 2024-03-26T20:51:10.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:10 smithi063 ceph-mon[24818]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-26T20:51:10.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:10 smithi063 ceph-mon[24818]: from='osd.0 [v2:172.21.15.63:6802/2118194723,v1:172.21.15.63:6803/2118194723]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-03-26T20:51:10.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:10 smithi174 ceph-mon[26934]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-26T20:51:10.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:10 smithi174 ceph-mon[26934]: from='osd.0 [v2:172.21.15.63:6802/2118194723,v1:172.21.15.63:6803/2118194723]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-03-26T20:51:11.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:11 smithi174 ceph-mon[26934]: from='osd.0 [v2:172.21.15.63:6802/2118194723,v1:172.21.15.63:6803/2118194723]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-03-26T20:51:11.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:11 smithi174 ceph-mon[26934]: osdmap e7: 1 total, 0 up, 1 in 2024-03-26T20:51:11.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:11 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-26T20:51:11.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:11 smithi174 ceph-mon[26934]: from='osd.0 [v2:172.21.15.63:6802/2118194723,v1:172.21.15.63:6803/2118194723]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi063", "root=default"]}]: dispatch 2024-03-26T20:51:11.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:11 smithi063 ceph-mon[24818]: from='osd.0 [v2:172.21.15.63:6802/2118194723,v1:172.21.15.63:6803/2118194723]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-03-26T20:51:11.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:11 smithi063 ceph-mon[24818]: osdmap e7: 1 total, 0 up, 1 in 2024-03-26T20:51:11.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:11 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-26T20:51:11.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:11 smithi063 ceph-mon[24818]: from='osd.0 [v2:172.21.15.63:6802/2118194723,v1:172.21.15.63:6803/2118194723]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi063", "root=default"]}]: dispatch 2024-03-26T20:51:12.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:12 smithi063 ceph-mon[24818]: from='osd.0 [v2:172.21.15.63:6802/2118194723,v1:172.21.15.63:6803/2118194723]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi063", "root=default"]}]': finished 2024-03-26T20:51:12.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:12 smithi063 ceph-mon[24818]: osdmap e8: 1 total, 0 up, 1 in 2024-03-26T20:51:12.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:12 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-26T20:51:12.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:12 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-26T20:51:12.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:12 smithi063 ceph-mon[24818]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-26T20:51:12.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:12 smithi174 ceph-mon[26934]: from='osd.0 [v2:172.21.15.63:6802/2118194723,v1:172.21.15.63:6803/2118194723]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi063", "root=default"]}]': finished 2024-03-26T20:51:12.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:12 smithi174 ceph-mon[26934]: osdmap e8: 1 total, 0 up, 1 in 2024-03-26T20:51:12.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:12 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-26T20:51:12.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:12 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-26T20:51:12.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:12 smithi174 ceph-mon[26934]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-26T20:51:13.522 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:13 smithi063 ceph-mon[24818]: purged_snaps scrub starts 2024-03-26T20:51:13.522 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:13 smithi063 ceph-mon[24818]: purged_snaps scrub ok 2024-03-26T20:51:13.522 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:13 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-26T20:51:13.523 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:13 smithi063 ceph-mon[24818]: osd.0 [v2:172.21.15.63:6802/2118194723,v1:172.21.15.63:6803/2118194723] boot 2024-03-26T20:51:13.523 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:13 smithi063 ceph-mon[24818]: osdmap e9: 1 total, 1 up, 1 in 2024-03-26T20:51:13.523 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:13 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-26T20:51:13.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:13 smithi174 ceph-mon[26934]: purged_snaps scrub starts 2024-03-26T20:51:13.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:13 smithi174 ceph-mon[26934]: purged_snaps scrub ok 2024-03-26T20:51:13.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:13 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-26T20:51:13.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:13 smithi174 ceph-mon[26934]: osd.0 [v2:172.21.15.63:6802/2118194723,v1:172.21.15.63:6803/2118194723] boot 2024-03-26T20:51:13.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:13 smithi174 ceph-mon[26934]: osdmap e9: 1 total, 1 up, 1 in 2024-03-26T20:51:13.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:13 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-26T20:51:14.442 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:14 smithi063 ceph-mon[24818]: osdmap e10: 1 total, 1 up, 1 in 2024-03-26T20:51:14.442 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:14 smithi063 ceph-mon[24818]: pgmap v31: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-03-26T20:51:14.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:14 smithi174 ceph-mon[26934]: osdmap e10: 1 total, 1 up, 1 in 2024-03-26T20:51:14.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:14 smithi174 ceph-mon[26934]: pgmap v31: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-03-26T20:51:15.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:15 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:51:15.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:15 smithi063 ceph-mon[24818]: Reconfiguring crash.smithi174 (monmap changed)... 2024-03-26T20:51:15.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:15 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi174", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-03-26T20:51:15.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:15 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:51:15.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:15 smithi063 ceph-mon[24818]: Reconfiguring daemon crash.smithi174 on smithi174 2024-03-26T20:51:15.861 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:15 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:51:15.861 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:15 smithi174 ceph-mon[26934]: Reconfiguring crash.smithi174 (monmap changed)... 2024-03-26T20:51:15.861 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:15 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi174", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-03-26T20:51:15.862 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:15 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:51:15.862 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:15 smithi174 ceph-mon[26934]: Reconfiguring daemon crash.smithi174 on smithi174 2024-03-26T20:51:16.140 INFO:teuthology.orchestra.run.smithi063.stdout:Created osd(s) 0 on host 'smithi063' 2024-03-26T20:51:16.798 DEBUG:teuthology.orchestra.run.smithi063:osd.0> sudo journalctl -f -n 0 -u ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715@osd.0.service 2024-03-26T20:51:16.800 INFO:tasks.cephadm:Deploying osd.1 on smithi063 with /dev/vg_nvme/lv_3... 2024-03-26T20:51:16.800 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-03-26T20:51:17.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:16 smithi063 ceph-mon[24818]: pgmap v32: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-03-26T20:51:17.052 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:16 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:51:17.052 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:16 smithi063 ceph-mon[24818]: Reconfiguring mgr.smithi174.sexoin (monmap changed)... 2024-03-26T20:51:17.052 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:16 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi174.sexoin", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-26T20:51:17.052 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:16 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-26T20:51:17.052 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:16 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:51:17.052 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:16 smithi063 ceph-mon[24818]: Reconfiguring daemon mgr.smithi174.sexoin on smithi174 2024-03-26T20:51:17.052 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:16 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:51:17.152 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:16 smithi174 ceph-mon[26934]: pgmap v32: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-03-26T20:51:17.152 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:16 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:51:17.152 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:16 smithi174 ceph-mon[26934]: Reconfiguring mgr.smithi174.sexoin (monmap changed)... 2024-03-26T20:51:17.152 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:16 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi174.sexoin", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-26T20:51:17.152 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:16 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-26T20:51:17.152 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:16 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:51:17.152 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:16 smithi174 ceph-mon[26934]: Reconfiguring daemon mgr.smithi174.sexoin on smithi174 2024-03-26T20:51:17.152 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:16 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:51:18.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:18 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:51:18.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:18 smithi174 ceph-mon[26934]: Reconfiguring mon.smithi174 (monmap changed)... 2024-03-26T20:51:18.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:18 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-26T20:51:18.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:18 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-26T20:51:18.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:18 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:51:18.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:18 smithi174 ceph-mon[26934]: Reconfiguring daemon mon.smithi174 on smithi174 2024-03-26T20:51:18.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:18 smithi174 ceph-mon[26934]: pgmap v33: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-03-26T20:51:18.330 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:18 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:51:18.330 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:18 smithi063 ceph-mon[24818]: Reconfiguring mon.smithi174 (monmap changed)... 2024-03-26T20:51:18.330 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:18 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-26T20:51:18.330 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:18 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-26T20:51:18.330 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:18 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:51:18.330 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:18 smithi063 ceph-mon[24818]: Reconfiguring daemon mon.smithi174 on smithi174 2024-03-26T20:51:18.330 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:18 smithi063 ceph-mon[24818]: pgmap v33: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-03-26T20:51:18.802 INFO:teuthology.orchestra.run.smithi063.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-03-26T20:51:18.802 INFO:teuthology.orchestra.run.smithi063.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-03-26T20:51:18.802 INFO:teuthology.orchestra.run.smithi063.stderr: stderr: 10+0 records in 2024-03-26T20:51:18.802 INFO:teuthology.orchestra.run.smithi063.stderr:10+0 records out 2024-03-26T20:51:18.802 INFO:teuthology.orchestra.run.smithi063.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0130726 s, 802 MB/s 2024-03-26T20:51:18.802 INFO:teuthology.orchestra.run.smithi063.stderr:--> Zapping successful for: 2024-03-26T20:51:19.621 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:19 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:51:19.621 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:19 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-26T20:51:19.621 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:19 smithi063 ceph-mon[24818]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-26T20:51:19.621 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:19 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.63:9093"}]: dispatch 2024-03-26T20:51:19.621 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:19 smithi063 ceph-mon[24818]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.63:9093"}]: dispatch 2024-03-26T20:51:19.621 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:19 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:51:19.621 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:19 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-26T20:51:19.621 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:19 smithi063 ceph-mon[24818]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-26T20:51:19.621 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:19 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.63:3000"}]: dispatch 2024-03-26T20:51:19.621 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:19 smithi063 ceph-mon[24818]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.63:3000"}]: dispatch 2024-03-26T20:51:19.621 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:19 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:51:19.621 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:19 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-26T20:51:19.621 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:19 smithi063 ceph-mon[24818]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-26T20:51:19.622 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:19 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.63:9095"}]: dispatch 2024-03-26T20:51:19.622 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:19 smithi063 ceph-mon[24818]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.63:9095"}]: dispatch 2024-03-26T20:51:19.622 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:19 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:51:19.622 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:19 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T20:51:19.622 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:19 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:51:19.622 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:19 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T20:51:19.796 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:19 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:51:19.796 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:19 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-26T20:51:19.796 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:19 smithi174 ceph-mon[26934]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-26T20:51:19.796 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:19 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.63:9093"}]: dispatch 2024-03-26T20:51:19.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:19 smithi174 ceph-mon[26934]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.63:9093"}]: dispatch 2024-03-26T20:51:19.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:19 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:51:19.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:19 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-26T20:51:19.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:19 smithi174 ceph-mon[26934]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-26T20:51:19.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:19 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.63:3000"}]: dispatch 2024-03-26T20:51:19.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:19 smithi174 ceph-mon[26934]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.63:3000"}]: dispatch 2024-03-26T20:51:19.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:19 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:51:19.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:19 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-26T20:51:19.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:19 smithi174 ceph-mon[26934]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-26T20:51:19.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:19 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.63:9095"}]: dispatch 2024-03-26T20:51:19.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:19 smithi174 ceph-mon[26934]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.63:9095"}]: dispatch 2024-03-26T20:51:19.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:19 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:51:19.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:19 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T20:51:19.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:19 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:51:19.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:19 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T20:51:19.925 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -- ceph orch daemon add osd smithi063:vg_nvme/lv_3 2024-03-26T20:51:20.501 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:20 smithi063 ceph-mon[24818]: pgmap v34: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-03-26T20:51:20.502 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:20 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:51:20.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:20 smithi174 ceph-mon[26934]: pgmap v34: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-03-26T20:51:20.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:20 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:51:22.657 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:22 smithi063 ceph-mon[24818]: pgmap v35: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-03-26T20:51:22.657 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:22 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-26T20:51:22.658 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:22 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-26T20:51:22.658 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:22 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:51:22.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:22 smithi174 ceph-mon[26934]: pgmap v35: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-03-26T20:51:22.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:22 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-26T20:51:22.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:22 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-26T20:51:22.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:22 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:51:23.551 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:23 smithi063 ceph-mon[24818]: from='client.14254 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi063:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:51:23.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:23 smithi174 ceph-mon[26934]: from='client.14254 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi063:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:51:24.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:24 smithi174 ceph-mon[26934]: pgmap v36: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-03-26T20:51:24.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:24 smithi063 ceph-mon[24818]: pgmap v36: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-03-26T20:51:25.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:25 smithi174 ceph-mon[26934]: from='client.? 172.21.15.63:0/2857194927' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cc56c3cf-5bdc-4d3b-b1fc-57f20f9c9509"}]: dispatch 2024-03-26T20:51:25.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:25 smithi174 ceph-mon[26934]: from='client.? 172.21.15.63:0/2857194927' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "cc56c3cf-5bdc-4d3b-b1fc-57f20f9c9509"}]': finished 2024-03-26T20:51:25.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:25 smithi174 ceph-mon[26934]: osdmap e11: 2 total, 1 up, 2 in 2024-03-26T20:51:25.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:25 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-26T20:51:25.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:25 smithi063 ceph-mon[24818]: from='client.? 172.21.15.63:0/2857194927' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cc56c3cf-5bdc-4d3b-b1fc-57f20f9c9509"}]: dispatch 2024-03-26T20:51:25.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:25 smithi063 ceph-mon[24818]: from='client.? 172.21.15.63:0/2857194927' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "cc56c3cf-5bdc-4d3b-b1fc-57f20f9c9509"}]': finished 2024-03-26T20:51:25.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:25 smithi063 ceph-mon[24818]: osdmap e11: 2 total, 1 up, 2 in 2024-03-26T20:51:25.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:25 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-26T20:51:26.784 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:26 smithi063 ceph-mon[24818]: pgmap v38: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-03-26T20:51:26.784 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:26 smithi063 ceph-mon[24818]: from='client.? 172.21.15.63:0/631124455' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-26T20:51:26.784 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:26 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:51:26.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:26 smithi174 ceph-mon[26934]: pgmap v38: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-03-26T20:51:26.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:26 smithi174 ceph-mon[26934]: from='client.? 172.21.15.63:0/631124455' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-26T20:51:26.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:26 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:51:28.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:28 smithi063 ceph-mon[24818]: pgmap v39: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-03-26T20:51:28.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:28 smithi174 ceph-mon[26934]: pgmap v39: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-03-26T20:51:30.743 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:30 smithi063 ceph-mon[24818]: pgmap v40: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-03-26T20:51:30.743 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:30 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:51:30.743 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:30 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-26T20:51:30.744 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:30 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:51:30.744 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:30 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:51:30.744 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:30 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T20:51:30.744 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:30 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:51:30.744 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:30 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T20:51:30.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:30 smithi174 ceph-mon[26934]: pgmap v40: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-03-26T20:51:30.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:30 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:51:30.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:30 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-26T20:51:30.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:30 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:51:30.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:30 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:51:30.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:30 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T20:51:30.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:30 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:51:30.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:30 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T20:51:31.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:31 smithi174 ceph-mon[26934]: Detected new or changed devices on smithi063 2024-03-26T20:51:31.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:31 smithi174 ceph-mon[26934]: Adjusting osd_memory_target on smithi063 to 12864M 2024-03-26T20:51:31.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:31 smithi063 ceph-mon[24818]: Detected new or changed devices on smithi063 2024-03-26T20:51:31.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:31 smithi063 ceph-mon[24818]: Adjusting osd_memory_target on smithi063 to 12864M 2024-03-26T20:51:32.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:32 smithi174 ceph-mon[26934]: pgmap v41: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-03-26T20:51:32.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:32 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:51:32.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:32 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-03-26T20:51:32.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:32 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:51:32.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:32 smithi063 ceph-mon[24818]: pgmap v41: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-03-26T20:51:32.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:32 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:51:32.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:32 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-03-26T20:51:32.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:32 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:51:33.715 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:33 smithi063 ceph-mon[24818]: Deploying daemon osd.1 on smithi063 2024-03-26T20:51:33.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:33 smithi174 ceph-mon[26934]: Deploying daemon osd.1 on smithi063 2024-03-26T20:51:34.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:34 smithi174 ceph-mon[26934]: pgmap v42: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-03-26T20:51:34.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:34 smithi063 ceph-mon[24818]: pgmap v42: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-03-26T20:51:36.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:36 smithi174 ceph-mon[26934]: pgmap v43: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-03-26T20:51:36.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:36 smithi063 ceph-mon[24818]: pgmap v43: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-03-26T20:51:38.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:38 smithi063 ceph-mon[24818]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-03-26T20:51:38.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:38 smithi174 ceph-mon[26934]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-03-26T20:51:39.809 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:39 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:51:39.809 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:39 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:51:39.809 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:39 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:51:39.809 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:39 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T20:51:39.809 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:39 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:51:39.809 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:39 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T20:51:39.866 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:39 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:51:39.867 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:39 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:51:39.867 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:39 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:51:39.867 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:39 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T20:51:39.867 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:39 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:51:39.867 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:39 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T20:51:40.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:40 smithi174 ceph-mon[26934]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-03-26T20:51:40.889 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:40 smithi063 ceph-mon[24818]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-03-26T20:51:42.029 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:41 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:51:42.029 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:41 smithi063 ceph-mon[24818]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-03-26T20:51:42.059 INFO:teuthology.orchestra.run.smithi063.stdout:Created osd(s) 1 on host 'smithi063' 2024-03-26T20:51:42.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:41 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:51:42.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:41 smithi174 ceph-mon[26934]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-03-26T20:51:42.725 DEBUG:teuthology.orchestra.run.smithi063:osd.1> sudo journalctl -f -n 0 -u ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715@osd.1.service 2024-03-26T20:51:42.727 INFO:tasks.cephadm:Deploying osd.2 on smithi063 with /dev/vg_nvme/lv_2... 2024-03-26T20:51:42.727 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-03-26T20:51:43.378 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:43 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:51:43.378 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:43 smithi063 ceph-mon[24818]: from='osd.1 [v2:172.21.15.63:6810/4265583190,v1:172.21.15.63:6811/4265583190]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-03-26T20:51:43.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:43 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:51:43.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:43 smithi174 ceph-mon[26934]: from='osd.1 [v2:172.21.15.63:6810/4265583190,v1:172.21.15.63:6811/4265583190]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-03-26T20:51:44.118 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:44 smithi063 ceph-mon[24818]: from='osd.1 [v2:172.21.15.63:6810/4265583190,v1:172.21.15.63:6811/4265583190]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-03-26T20:51:44.118 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:44 smithi063 ceph-mon[24818]: osdmap e12: 2 total, 1 up, 2 in 2024-03-26T20:51:44.118 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:44 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-26T20:51:44.118 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:44 smithi063 ceph-mon[24818]: from='osd.1 [v2:172.21.15.63:6810/4265583190,v1:172.21.15.63:6811/4265583190]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi063", "root=default"]}]: dispatch 2024-03-26T20:51:44.118 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:44 smithi063 ceph-mon[24818]: pgmap v48: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-03-26T20:51:44.118 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 20:51:44 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-1[39331]: 2024-03-26T20:51:44.072+0000 7fbfdd35c700 -1 osd.1 0 waiting for initial osdmap 2024-03-26T20:51:44.118 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 20:51:44 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-1[39331]: 2024-03-26T20:51:44.079+0000 7fbfd9cf7700 -1 osd.1 13 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-26T20:51:44.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:44 smithi174 ceph-mon[26934]: from='osd.1 [v2:172.21.15.63:6810/4265583190,v1:172.21.15.63:6811/4265583190]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-03-26T20:51:44.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:44 smithi174 ceph-mon[26934]: osdmap e12: 2 total, 1 up, 2 in 2024-03-26T20:51:44.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:44 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-26T20:51:44.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:44 smithi174 ceph-mon[26934]: from='osd.1 [v2:172.21.15.63:6810/4265583190,v1:172.21.15.63:6811/4265583190]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi063", "root=default"]}]: dispatch 2024-03-26T20:51:44.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:44 smithi174 ceph-mon[26934]: pgmap v48: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-03-26T20:51:45.471 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:45 smithi063 ceph-mon[24818]: from='osd.1 [v2:172.21.15.63:6810/4265583190,v1:172.21.15.63:6811/4265583190]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi063", "root=default"]}]': finished 2024-03-26T20:51:45.471 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:45 smithi063 ceph-mon[24818]: osdmap e13: 2 total, 1 up, 2 in 2024-03-26T20:51:45.471 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:45 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-26T20:51:45.471 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:45 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-26T20:51:45.471 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:45 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-26T20:51:45.472 INFO:teuthology.orchestra.run.smithi063.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-03-26T20:51:45.472 INFO:teuthology.orchestra.run.smithi063.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-03-26T20:51:45.472 INFO:teuthology.orchestra.run.smithi063.stderr: stderr: 10+0 records in 2024-03-26T20:51:45.472 INFO:teuthology.orchestra.run.smithi063.stderr:10+0 records out 2024-03-26T20:51:45.472 INFO:teuthology.orchestra.run.smithi063.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0123907 s, 846 MB/s 2024-03-26T20:51:45.472 INFO:teuthology.orchestra.run.smithi063.stderr:--> Zapping successful for: 2024-03-26T20:51:45.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:45 smithi174 ceph-mon[26934]: from='osd.1 [v2:172.21.15.63:6810/4265583190,v1:172.21.15.63:6811/4265583190]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi063", "root=default"]}]': finished 2024-03-26T20:51:45.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:45 smithi174 ceph-mon[26934]: osdmap e13: 2 total, 1 up, 2 in 2024-03-26T20:51:45.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:45 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-26T20:51:45.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:45 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-26T20:51:45.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:45 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-26T20:51:46.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:46 smithi063 ceph-mon[24818]: purged_snaps scrub starts 2024-03-26T20:51:46.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:46 smithi063 ceph-mon[24818]: purged_snaps scrub ok 2024-03-26T20:51:46.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:46 smithi063 ceph-mon[24818]: osd.1 [v2:172.21.15.63:6810/4265583190,v1:172.21.15.63:6811/4265583190] boot 2024-03-26T20:51:46.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:46 smithi063 ceph-mon[24818]: osdmap e14: 2 total, 2 up, 2 in 2024-03-26T20:51:46.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:46 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-26T20:51:46.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:46 smithi063 ceph-mon[24818]: pgmap v51: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-03-26T20:51:46.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:46 smithi174 ceph-mon[26934]: purged_snaps scrub starts 2024-03-26T20:51:46.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:46 smithi174 ceph-mon[26934]: purged_snaps scrub ok 2024-03-26T20:51:46.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:46 smithi174 ceph-mon[26934]: osd.1 [v2:172.21.15.63:6810/4265583190,v1:172.21.15.63:6811/4265583190] boot 2024-03-26T20:51:46.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:46 smithi174 ceph-mon[26934]: osdmap e14: 2 total, 2 up, 2 in 2024-03-26T20:51:46.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:46 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-26T20:51:46.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:46 smithi174 ceph-mon[26934]: pgmap v51: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-03-26T20:51:46.634 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -- ceph orch daemon add osd smithi063:vg_nvme/lv_2 2024-03-26T20:51:47.179 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:47 smithi063 ceph-mon[24818]: osdmap e15: 2 total, 2 up, 2 in 2024-03-26T20:51:47.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:47 smithi174 ceph-mon[26934]: osdmap e15: 2 total, 2 up, 2 in 2024-03-26T20:51:48.405 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:48 smithi063 ceph-mon[24818]: pgmap v53: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-03-26T20:51:48.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:48 smithi174 ceph-mon[26934]: pgmap v53: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-03-26T20:51:49.352 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:49 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-26T20:51:49.352 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:49 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-26T20:51:49.352 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:49 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:51:49.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:49 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-26T20:51:49.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:49 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-26T20:51:49.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:49 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:51:50.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:50 smithi063 ceph-mon[24818]: from='client.14272 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi063:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:51:50.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:50 smithi063 ceph-mon[24818]: pgmap v54: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-26T20:51:50.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:50 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:51:51.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:50 smithi174 ceph-mon[26934]: from='client.14272 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi063:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:51:51.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:50 smithi174 ceph-mon[26934]: pgmap v54: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-26T20:51:51.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:50 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:51:52.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:52 smithi063 ceph-mon[24818]: pgmap v55: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-26T20:51:52.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:52 smithi063 ceph-mon[24818]: from='client.? 172.21.15.63:0/1816024048' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5dcc41b7-668e-46c4-a389-e8a6cf2a5f64"}]: dispatch 2024-03-26T20:51:52.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:52 smithi063 ceph-mon[24818]: from='client.? 172.21.15.63:0/1816024048' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "5dcc41b7-668e-46c4-a389-e8a6cf2a5f64"}]': finished 2024-03-26T20:51:52.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:52 smithi063 ceph-mon[24818]: osdmap e16: 3 total, 2 up, 3 in 2024-03-26T20:51:52.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:52 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-26T20:51:53.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:52 smithi174 ceph-mon[26934]: pgmap v55: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-26T20:51:53.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:52 smithi174 ceph-mon[26934]: from='client.? 172.21.15.63:0/1816024048' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5dcc41b7-668e-46c4-a389-e8a6cf2a5f64"}]: dispatch 2024-03-26T20:51:53.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:52 smithi174 ceph-mon[26934]: from='client.? 172.21.15.63:0/1816024048' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "5dcc41b7-668e-46c4-a389-e8a6cf2a5f64"}]': finished 2024-03-26T20:51:53.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:52 smithi174 ceph-mon[26934]: osdmap e16: 3 total, 2 up, 3 in 2024-03-26T20:51:53.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:52 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-26T20:51:54.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:53 smithi174 ceph-mon[26934]: from='client.? 172.21.15.63:0/1006190090' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-26T20:51:54.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:53 smithi063 ceph-mon[24818]: from='client.? 172.21.15.63:0/1006190090' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-26T20:51:54.947 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:54 smithi063 ceph-mon[24818]: pgmap v57: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-26T20:51:54.949 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:54 smithi174 ceph-mon[26934]: pgmap v57: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-26T20:51:56.973 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:56 smithi063 ceph-mon[24818]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-26T20:51:57.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:56 smithi174 ceph-mon[26934]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-26T20:51:59.052 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:58 smithi063 ceph-mon[24818]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-26T20:51:59.052 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:58 smithi063 ceph-mon[24818]: Detected new or changed devices on smithi063 2024-03-26T20:51:59.052 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:58 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:51:59.052 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:58 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-26T20:51:59.052 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:58 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-26T20:51:59.052 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:58 smithi063 ceph-mon[24818]: Adjusting osd_memory_target on smithi063 to 6432M 2024-03-26T20:51:59.052 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:58 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:51:59.052 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:58 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:51:59.052 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:58 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T20:51:59.053 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:58 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:51:59.053 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:51:58 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T20:51:59.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:58 smithi174 ceph-mon[26934]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-26T20:51:59.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:58 smithi174 ceph-mon[26934]: Detected new or changed devices on smithi063 2024-03-26T20:51:59.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:58 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:51:59.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:58 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-26T20:51:59.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:58 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-26T20:51:59.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:58 smithi174 ceph-mon[26934]: Adjusting osd_memory_target on smithi063 to 6432M 2024-03-26T20:51:59.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:58 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:51:59.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:58 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:51:59.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:58 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T20:51:59.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:58 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:51:59.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:51:58 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T20:52:01.178 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:00 smithi063 ceph-mon[24818]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-26T20:52:01.178 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:00 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-03-26T20:52:01.179 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:00 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:52:01.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:00 smithi174 ceph-mon[26934]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-26T20:52:01.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:00 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-03-26T20:52:01.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:00 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:52:02.125 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:01 smithi063 ceph-mon[24818]: Deploying daemon osd.2 on smithi063 2024-03-26T20:52:02.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:01 smithi174 ceph-mon[26934]: Deploying daemon osd.2 on smithi063 2024-03-26T20:52:03.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:02 smithi174 ceph-mon[26934]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-26T20:52:03.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:02 smithi063 ceph-mon[24818]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-26T20:52:05.097 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:04 smithi063 ceph-mon[24818]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-26T20:52:05.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:04 smithi174 ceph-mon[26934]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-26T20:52:06.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:06 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:52:06.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:06 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:52:06.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:06 smithi174 ceph-mon[26934]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-26T20:52:06.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:06 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:52:06.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:06 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:52:06.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:06 smithi063 ceph-mon[24818]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-26T20:52:08.559 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:08 smithi063 ceph-mon[24818]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-26T20:52:08.559 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:08 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi063.lxhujt/mirror_snapshot_schedule"}]: dispatch 2024-03-26T20:52:08.559 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:08 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi063.lxhujt/trash_purge_schedule"}]: dispatch 2024-03-26T20:52:08.560 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:08 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:52:08.560 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:08 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T20:52:08.560 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:08 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:52:08.560 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:08 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T20:52:08.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:08 smithi174 ceph-mon[26934]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-26T20:52:08.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:08 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi063.lxhujt/mirror_snapshot_schedule"}]: dispatch 2024-03-26T20:52:08.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:08 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi063.lxhujt/trash_purge_schedule"}]: dispatch 2024-03-26T20:52:08.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:08 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:52:08.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:08 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T20:52:08.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:08 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:52:08.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:08 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T20:52:10.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:10 smithi174 ceph-mon[26934]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-26T20:52:10.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:10 smithi063 ceph-mon[24818]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-26T20:52:11.618 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:11 smithi063 ceph-mon[24818]: from='osd.2 [v2:172.21.15.63:6818/2962020712,v1:172.21.15.63:6819/2962020712]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-03-26T20:52:11.653 INFO:teuthology.orchestra.run.smithi063.stdout:Created osd(s) 2 on host 'smithi063' 2024-03-26T20:52:11.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:11 smithi174 ceph-mon[26934]: from='osd.2 [v2:172.21.15.63:6818/2962020712,v1:172.21.15.63:6819/2962020712]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-03-26T20:52:12.325 DEBUG:teuthology.orchestra.run.smithi063:osd.2> sudo journalctl -f -n 0 -u ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715@osd.2.service 2024-03-26T20:52:12.328 INFO:tasks.cephadm:Deploying osd.3 on smithi063 with /dev/vg_nvme/lv_1... 2024-03-26T20:52:12.328 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-03-26T20:52:12.551 INFO:journalctl@ceph.osd.2.smithi063.stdout:Mar 26 20:52:12 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-2[43070]: 2024-03-26T20:52:12.541+0000 7f63f9c27700 -1 osd.2 0 waiting for initial osdmap 2024-03-26T20:52:12.552 INFO:journalctl@ceph.osd.2.smithi063.stdout:Mar 26 20:52:12 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-2[43070]: 2024-03-26T20:52:12.545+0000 7f63f45be700 -1 osd.2 18 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-26T20:52:12.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:12 smithi063 ceph-mon[24818]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-26T20:52:12.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:12 smithi063 ceph-mon[24818]: from='osd.2 [v2:172.21.15.63:6818/2962020712,v1:172.21.15.63:6819/2962020712]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-03-26T20:52:12.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:12 smithi063 ceph-mon[24818]: osdmap e17: 3 total, 2 up, 3 in 2024-03-26T20:52:12.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:12 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-26T20:52:12.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:12 smithi063 ceph-mon[24818]: from='osd.2 [v2:172.21.15.63:6818/2962020712,v1:172.21.15.63:6819/2962020712]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi063", "root=default"]}]: dispatch 2024-03-26T20:52:12.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:12 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:52:12.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:12 smithi174 ceph-mon[26934]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-03-26T20:52:12.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:12 smithi174 ceph-mon[26934]: from='osd.2 [v2:172.21.15.63:6818/2962020712,v1:172.21.15.63:6819/2962020712]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-03-26T20:52:12.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:12 smithi174 ceph-mon[26934]: osdmap e17: 3 total, 2 up, 3 in 2024-03-26T20:52:12.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:12 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-26T20:52:12.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:12 smithi174 ceph-mon[26934]: from='osd.2 [v2:172.21.15.63:6818/2962020712,v1:172.21.15.63:6819/2962020712]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi063", "root=default"]}]: dispatch 2024-03-26T20:52:12.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:12 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:52:13.880 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:13 smithi063 ceph-mon[24818]: from='osd.2 [v2:172.21.15.63:6818/2962020712,v1:172.21.15.63:6819/2962020712]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi063", "root=default"]}]': finished 2024-03-26T20:52:13.881 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:13 smithi063 ceph-mon[24818]: osdmap e18: 3 total, 2 up, 3 in 2024-03-26T20:52:13.881 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:13 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-26T20:52:13.881 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:13 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-26T20:52:14.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:13 smithi174 ceph-mon[26934]: from='osd.2 [v2:172.21.15.63:6818/2962020712,v1:172.21.15.63:6819/2962020712]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi063", "root=default"]}]': finished 2024-03-26T20:52:14.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:13 smithi174 ceph-mon[26934]: osdmap e18: 3 total, 2 up, 3 in 2024-03-26T20:52:14.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:13 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-26T20:52:14.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:13 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-26T20:52:15.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:14 smithi174 ceph-mon[26934]: purged_snaps scrub starts 2024-03-26T20:52:15.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:14 smithi174 ceph-mon[26934]: purged_snaps scrub ok 2024-03-26T20:52:15.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:14 smithi174 ceph-mon[26934]: pgmap v69: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2024-03-26T20:52:15.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:14 smithi174 ceph-mon[26934]: osd.2 [v2:172.21.15.63:6818/2962020712,v1:172.21.15.63:6819/2962020712] boot 2024-03-26T20:52:15.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:14 smithi174 ceph-mon[26934]: osdmap e19: 3 total, 3 up, 3 in 2024-03-26T20:52:15.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:14 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-26T20:52:15.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:14 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-03-26T20:52:15.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:14 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-03-26T20:52:15.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:14 smithi174 ceph-mon[26934]: osdmap e20: 3 total, 3 up, 3 in 2024-03-26T20:52:15.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:14 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-03-26T20:52:15.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:14 smithi063 ceph-mon[24818]: purged_snaps scrub starts 2024-03-26T20:52:15.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:14 smithi063 ceph-mon[24818]: purged_snaps scrub ok 2024-03-26T20:52:15.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:14 smithi063 ceph-mon[24818]: pgmap v69: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2024-03-26T20:52:15.052 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:14 smithi063 ceph-mon[24818]: osd.2 [v2:172.21.15.63:6818/2962020712,v1:172.21.15.63:6819/2962020712] boot 2024-03-26T20:52:15.052 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:14 smithi063 ceph-mon[24818]: osdmap e19: 3 total, 3 up, 3 in 2024-03-26T20:52:15.052 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:14 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-26T20:52:15.052 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:14 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-03-26T20:52:15.052 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:14 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-03-26T20:52:15.052 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:14 smithi063 ceph-mon[24818]: osdmap e20: 3 total, 3 up, 3 in 2024-03-26T20:52:15.052 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:14 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-03-26T20:52:16.259 INFO:teuthology.orchestra.run.smithi063.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-03-26T20:52:16.259 INFO:teuthology.orchestra.run.smithi063.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-03-26T20:52:16.259 INFO:teuthology.orchestra.run.smithi063.stderr: stderr: 10+0 records in 2024-03-26T20:52:16.259 INFO:teuthology.orchestra.run.smithi063.stderr:10+0 records out 2024-03-26T20:52:16.259 INFO:teuthology.orchestra.run.smithi063.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.013439 s, 780 MB/s 2024-03-26T20:52:16.259 INFO:teuthology.orchestra.run.smithi063.stderr:--> Zapping successful for: 2024-03-26T20:52:16.500 INFO:journalctl@ceph.osd.0.smithi063.stdout:Mar 26 20:52:16 smithi063 sudo[44437]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-03-26T20:52:16.501 INFO:journalctl@ceph.osd.0.smithi063.stdout:Mar 26 20:52:16 smithi063 sudo[44437]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-03-26T20:52:16.501 INFO:journalctl@ceph.osd.0.smithi063.stdout:Mar 26 20:52:16 smithi063 sudo[44437]: pam_unix(sudo:session): session closed for user root 2024-03-26T20:52:16.784 INFO:journalctl@ceph.osd.0.smithi063.stdout:Mar 26 20:52:16 smithi063 sudo[44465]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-03-26T20:52:16.785 INFO:journalctl@ceph.osd.0.smithi063.stdout:Mar 26 20:52:16 smithi063 sudo[44465]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-03-26T20:52:16.785 INFO:journalctl@ceph.osd.0.smithi063.stdout:Mar 26 20:52:16 smithi063 sudo[44465]: pam_unix(sudo:session): session closed for user root 2024-03-26T20:52:16.785 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:16 smithi063 ceph-mon[24818]: pgmap v72: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-03-26T20:52:16.785 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:16 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-03-26T20:52:16.785 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:16 smithi063 ceph-mon[24818]: osdmap e21: 3 total, 3 up, 3 in 2024-03-26T20:52:16.785 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:16 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:52:16.785 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:16 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:52:16.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:16 smithi174 ceph-mon[26934]: pgmap v72: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-03-26T20:52:16.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:16 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-03-26T20:52:16.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:16 smithi174 ceph-mon[26934]: osdmap e21: 3 total, 3 up, 3 in 2024-03-26T20:52:16.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:16 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:52:16.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:16 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:52:16.930 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -- ceph orch daemon add osd smithi063:vg_nvme/lv_1 2024-03-26T20:52:17.051 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 20:52:16 smithi063 sudo[44484]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-03-26T20:52:17.052 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 20:52:16 smithi063 sudo[44484]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-03-26T20:52:17.052 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 20:52:16 smithi063 sudo[44484]: pam_unix(sudo:session): session closed for user root 2024-03-26T20:52:17.310 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 20:52:17 smithi063 sudo[44519]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-03-26T20:52:17.311 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 20:52:17 smithi063 sudo[44519]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-03-26T20:52:17.311 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 20:52:17 smithi063 sudo[44519]: pam_unix(sudo:session): session closed for user root 2024-03-26T20:52:17.591 INFO:journalctl@ceph.osd.2.smithi063.stdout:Mar 26 20:52:17 smithi063 sudo[44556]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-03-26T20:52:17.591 INFO:journalctl@ceph.osd.2.smithi063.stdout:Mar 26 20:52:17 smithi063 sudo[44556]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-03-26T20:52:17.592 INFO:journalctl@ceph.osd.2.smithi063.stdout:Mar 26 20:52:17 smithi063 sudo[44556]: pam_unix(sudo:session): session closed for user root 2024-03-26T20:52:17.853 INFO:journalctl@ceph.osd.2.smithi063.stdout:Mar 26 20:52:17 smithi063 sudo[44571]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-03-26T20:52:17.854 INFO:journalctl@ceph.osd.2.smithi063.stdout:Mar 26 20:52:17 smithi063 sudo[44571]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-03-26T20:52:17.854 INFO:journalctl@ceph.osd.2.smithi063.stdout:Mar 26 20:52:17 smithi063 sudo[44571]: pam_unix(sudo:session): session closed for user root 2024-03-26T20:52:18.115 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:17 smithi063 sudo[44575]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-03-26T20:52:18.115 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:17 smithi063 sudo[44575]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-03-26T20:52:18.115 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:17 smithi063 sudo[44575]: pam_unix(sudo:session): session closed for user root 2024-03-26T20:52:18.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:18 smithi174 ceph-mon[26934]: osdmap e22: 3 total, 3 up, 3 in 2024-03-26T20:52:18.505 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:18 smithi063 sudo[44594]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-03-26T20:52:18.505 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:18 smithi063 sudo[44594]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-03-26T20:52:18.505 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:18 smithi063 sudo[44594]: pam_unix(sudo:session): session closed for user root 2024-03-26T20:52:18.505 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:18 smithi063 ceph-mon[24818]: osdmap e22: 3 total, 3 up, 3 in 2024-03-26T20:52:18.626 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:18 smithi174 sudo[29258]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-03-26T20:52:18.626 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:18 smithi174 sudo[29258]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-03-26T20:52:18.626 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:18 smithi174 sudo[29258]: pam_unix(sudo:session): session closed for user root 2024-03-26T20:52:19.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:18 smithi174 sudo[29261]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-03-26T20:52:19.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:18 smithi174 sudo[29261]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-03-26T20:52:19.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:18 smithi174 sudo[29261]: pam_unix(sudo:session): session closed for user root 2024-03-26T20:52:19.399 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:19 smithi063 ceph-mon[24818]: pgmap v75: 1 pgs: 1 active+clean; 449 KiB data, 15 MiB used, 268 GiB / 268 GiB avail 2024-03-26T20:52:19.400 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:19 smithi063 ceph-mon[24818]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-03-26T20:52:19.400 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:19 smithi063 ceph-mon[24818]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-03-26T20:52:19.400 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:19 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mon metadata", "id": "smithi063"}]: dispatch 2024-03-26T20:52:19.400 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:19 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mon metadata", "id": "smithi174"}]: dispatch 2024-03-26T20:52:19.400 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:19 smithi063 ceph-mon[24818]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-03-26T20:52:19.400 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:19 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mon metadata", "id": "smithi063"}]: dispatch 2024-03-26T20:52:19.400 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:19 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mon metadata", "id": "smithi174"}]: dispatch 2024-03-26T20:52:19.400 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:19 smithi063 ceph-mon[24818]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-03-26T20:52:19.400 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:19 smithi063 ceph-mon[24818]: mgrmap e19: smithi063.lxhujt(active, since 2m), standbys: smithi174.sexoin 2024-03-26T20:52:19.400 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:19 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:52:19.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:19 smithi174 ceph-mon[26934]: pgmap v75: 1 pgs: 1 active+clean; 449 KiB data, 15 MiB used, 268 GiB / 268 GiB avail 2024-03-26T20:52:19.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:19 smithi174 ceph-mon[26934]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-03-26T20:52:19.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:19 smithi174 ceph-mon[26934]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-03-26T20:52:19.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:19 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mon metadata", "id": "smithi063"}]: dispatch 2024-03-26T20:52:19.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:19 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mon metadata", "id": "smithi174"}]: dispatch 2024-03-26T20:52:19.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:19 smithi174 ceph-mon[26934]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-03-26T20:52:19.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:19 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mon metadata", "id": "smithi063"}]: dispatch 2024-03-26T20:52:19.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:19 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mon metadata", "id": "smithi174"}]: dispatch 2024-03-26T20:52:19.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:19 smithi174 ceph-mon[26934]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-03-26T20:52:19.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:19 smithi174 ceph-mon[26934]: mgrmap e19: smithi063.lxhujt(active, since 2m), standbys: smithi174.sexoin 2024-03-26T20:52:19.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:19 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:52:20.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:20 smithi063 ceph-mon[24818]: pgmap v76: 1 pgs: 1 active+clean; 449 KiB data, 16 MiB used, 268 GiB / 268 GiB avail 2024-03-26T20:52:20.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:20 smithi174 ceph-mon[26934]: pgmap v76: 1 pgs: 1 active+clean; 449 KiB data, 16 MiB used, 268 GiB / 268 GiB avail 2024-03-26T20:52:21.178 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:21 smithi063 ceph-mon[24818]: from='client.14296 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi063:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:52:21.178 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:21 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-26T20:52:21.178 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:21 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-26T20:52:21.179 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:21 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:52:21.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:21 smithi174 ceph-mon[26934]: from='client.14296 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi063:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:52:21.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:21 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-26T20:52:21.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:21 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-26T20:52:21.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:21 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:52:22.283 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:22 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:52:22.283 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:22 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-26T20:52:22.283 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:22 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-26T20:52:22.284 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:22 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-26T20:52:22.284 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:22 smithi063 ceph-mon[24818]: Adjusting osd_memory_target on smithi063 to 4288M 2024-03-26T20:52:22.284 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:22 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:52:22.284 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:22 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:52:22.284 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:22 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T20:52:22.284 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:22 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:52:22.284 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:22 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T20:52:22.284 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:22 smithi063 ceph-mon[24818]: pgmap v77: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 5.8 KiB/s rd, 279 KiB/s wr, 19 op/s 2024-03-26T20:52:22.284 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:22 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:52:22.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:22 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:52:22.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:22 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-26T20:52:22.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:22 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-26T20:52:22.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:22 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-26T20:52:22.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:22 smithi174 ceph-mon[26934]: Adjusting osd_memory_target on smithi063 to 4288M 2024-03-26T20:52:22.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:22 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:52:22.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:22 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:52:22.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:22 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T20:52:22.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:22 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:52:22.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:22 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T20:52:22.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:22 smithi174 ceph-mon[26934]: pgmap v77: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 5.8 KiB/s rd, 279 KiB/s wr, 19 op/s 2024-03-26T20:52:22.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:22 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:52:23.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:23 smithi174 ceph-mon[26934]: from='client.? 172.21.15.63:0/2096028235' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "21eb7156-f9a5-4108-982d-47847322b16e"}]: dispatch 2024-03-26T20:52:23.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:23 smithi174 ceph-mon[26934]: from='client.? 172.21.15.63:0/2096028235' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "21eb7156-f9a5-4108-982d-47847322b16e"}]': finished 2024-03-26T20:52:23.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:23 smithi174 ceph-mon[26934]: osdmap e23: 4 total, 3 up, 4 in 2024-03-26T20:52:23.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:23 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-26T20:52:23.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:23 smithi063 ceph-mon[24818]: from='client.? 172.21.15.63:0/2096028235' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "21eb7156-f9a5-4108-982d-47847322b16e"}]: dispatch 2024-03-26T20:52:23.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:23 smithi063 ceph-mon[24818]: from='client.? 172.21.15.63:0/2096028235' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "21eb7156-f9a5-4108-982d-47847322b16e"}]': finished 2024-03-26T20:52:23.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:23 smithi063 ceph-mon[24818]: osdmap e23: 4 total, 3 up, 4 in 2024-03-26T20:52:23.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:23 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-26T20:52:24.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:24 smithi063 ceph-mon[24818]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 4.0 KiB/s rd, 196 KiB/s wr, 13 op/s 2024-03-26T20:52:24.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:24 smithi063 ceph-mon[24818]: from='client.? 172.21.15.63:0/4106794842' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-26T20:52:24.699 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:24 smithi174 ceph-mon[26934]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 4.0 KiB/s rd, 196 KiB/s wr, 13 op/s 2024-03-26T20:52:24.700 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:24 smithi174 ceph-mon[26934]: from='client.? 172.21.15.63:0/4106794842' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-26T20:52:26.659 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:26 smithi063 ceph-mon[24818]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 3.1 KiB/s rd, 151 KiB/s wr, 10 op/s 2024-03-26T20:52:26.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:26 smithi174 ceph-mon[26934]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 3.1 KiB/s rd, 151 KiB/s wr, 10 op/s 2024-03-26T20:52:28.358 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:28 smithi063 ceph-mon[24818]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.7 KiB/s rd, 131 KiB/s wr, 8 op/s 2024-03-26T20:52:28.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:28 smithi174 ceph-mon[26934]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.7 KiB/s rd, 131 KiB/s wr, 8 op/s 2024-03-26T20:52:30.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:30 smithi174 ceph-mon[26934]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.7 KiB/s rd, 131 KiB/s wr, 8 op/s 2024-03-26T20:52:30.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:30 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-03-26T20:52:30.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:30 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:52:30.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:30 smithi063 ceph-mon[24818]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.7 KiB/s rd, 131 KiB/s wr, 8 op/s 2024-03-26T20:52:30.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:30 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-03-26T20:52:30.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:30 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:52:31.586 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:31 smithi063 ceph-mon[24818]: Deploying daemon osd.3 on smithi063 2024-03-26T20:52:31.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:31 smithi174 ceph-mon[26934]: Deploying daemon osd.3 on smithi063 2024-03-26T20:52:32.789 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:32 smithi063 ceph-mon[24818]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-03-26T20:52:32.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:32 smithi174 ceph-mon[26934]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-03-26T20:52:34.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:34 smithi174 ceph-mon[26934]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-03-26T20:52:34.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:34 smithi063 ceph-mon[24818]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-03-26T20:52:36.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:36 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:52:36.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:36 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T20:52:36.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:36 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:52:36.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:36 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T20:52:36.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:36 smithi174 ceph-mon[26934]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-03-26T20:52:36.551 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:36 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:52:36.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:36 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T20:52:36.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:36 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:52:36.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:36 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T20:52:36.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:36 smithi063 ceph-mon[24818]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-03-26T20:52:38.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:38 smithi063 ceph-mon[24818]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-03-26T20:52:38.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:38 smithi063 ceph-mon[24818]: from='osd.3 [v2:172.21.15.63:6826/284512069,v1:172.21.15.63:6827/284512069]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-03-26T20:52:38.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:38 smithi174 ceph-mon[26934]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-03-26T20:52:38.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:38 smithi174 ceph-mon[26934]: from='osd.3 [v2:172.21.15.63:6826/284512069,v1:172.21.15.63:6827/284512069]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-03-26T20:52:39.108 INFO:teuthology.orchestra.run.smithi063.stdout:Created osd(s) 3 on host 'smithi063' 2024-03-26T20:52:39.333 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:39 smithi063 ceph-mon[24818]: from='osd.3 [v2:172.21.15.63:6826/284512069,v1:172.21.15.63:6827/284512069]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-03-26T20:52:39.333 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:39 smithi063 ceph-mon[24818]: osdmap e24: 4 total, 3 up, 4 in 2024-03-26T20:52:39.333 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:39 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-26T20:52:39.333 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:39 smithi063 ceph-mon[24818]: from='osd.3 [v2:172.21.15.63:6826/284512069,v1:172.21.15.63:6827/284512069]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi063", "root=default"]}]: dispatch 2024-03-26T20:52:39.333 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:39 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:52:39.748 DEBUG:teuthology.orchestra.run.smithi063:osd.3> sudo journalctl -f -n 0 -u ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715@osd.3.service 2024-03-26T20:52:39.751 INFO:tasks.cephadm:Deploying osd.4 on smithi174 with /dev/vg_nvme/lv_4... 2024-03-26T20:52:39.751 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-03-26T20:52:39.781 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:39 smithi174 ceph-mon[26934]: from='osd.3 [v2:172.21.15.63:6826/284512069,v1:172.21.15.63:6827/284512069]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-03-26T20:52:39.781 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:39 smithi174 ceph-mon[26934]: osdmap e24: 4 total, 3 up, 4 in 2024-03-26T20:52:39.781 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:39 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-26T20:52:39.781 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:39 smithi174 ceph-mon[26934]: from='osd.3 [v2:172.21.15.63:6826/284512069,v1:172.21.15.63:6827/284512069]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi063", "root=default"]}]: dispatch 2024-03-26T20:52:39.781 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:39 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:52:40.705 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:40 smithi063 ceph-mon[24818]: from='osd.3 [v2:172.21.15.63:6826/284512069,v1:172.21.15.63:6827/284512069]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi063", "root=default"]}]': finished 2024-03-26T20:52:40.705 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:40 smithi063 ceph-mon[24818]: osdmap e25: 4 total, 3 up, 4 in 2024-03-26T20:52:40.705 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:40 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-26T20:52:40.705 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:40 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-26T20:52:40.706 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:40 smithi063 ceph-mon[24818]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-03-26T20:52:40.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:40 smithi174 ceph-mon[26934]: from='osd.3 [v2:172.21.15.63:6826/284512069,v1:172.21.15.63:6827/284512069]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi063", "root=default"]}]': finished 2024-03-26T20:52:40.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:40 smithi174 ceph-mon[26934]: osdmap e25: 4 total, 3 up, 4 in 2024-03-26T20:52:40.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:40 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-26T20:52:40.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:40 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-26T20:52:40.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:40 smithi174 ceph-mon[26934]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-03-26T20:52:41.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:41 smithi174 ceph-mon[26934]: purged_snaps scrub starts 2024-03-26T20:52:41.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:41 smithi174 ceph-mon[26934]: purged_snaps scrub ok 2024-03-26T20:52:41.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:41 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-26T20:52:41.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:41 smithi174 ceph-mon[26934]: osd.3 [v2:172.21.15.63:6826/284512069,v1:172.21.15.63:6827/284512069] boot 2024-03-26T20:52:41.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:41 smithi174 ceph-mon[26934]: osdmap e26: 4 total, 4 up, 4 in 2024-03-26T20:52:41.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:41 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-26T20:52:41.648 INFO:teuthology.orchestra.run.smithi174.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-03-26T20:52:41.648 INFO:teuthology.orchestra.run.smithi174.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-03-26T20:52:41.649 INFO:teuthology.orchestra.run.smithi174.stderr: stderr: 10+0 records in 2024-03-26T20:52:41.649 INFO:teuthology.orchestra.run.smithi174.stderr:10+0 records out 2024-03-26T20:52:41.649 INFO:teuthology.orchestra.run.smithi174.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0131767 s, 796 MB/s 2024-03-26T20:52:41.649 INFO:teuthology.orchestra.run.smithi174.stderr:--> Zapping successful for: 2024-03-26T20:52:41.661 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:41 smithi063 ceph-mon[24818]: purged_snaps scrub starts 2024-03-26T20:52:41.661 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:41 smithi063 ceph-mon[24818]: purged_snaps scrub ok 2024-03-26T20:52:41.661 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:41 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-26T20:52:41.661 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:41 smithi063 ceph-mon[24818]: osd.3 [v2:172.21.15.63:6826/284512069,v1:172.21.15.63:6827/284512069] boot 2024-03-26T20:52:41.661 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:41 smithi063 ceph-mon[24818]: osdmap e26: 4 total, 4 up, 4 in 2024-03-26T20:52:41.661 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:41 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-26T20:52:42.373 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -- ceph orch daemon add osd smithi174:vg_nvme/lv_4 2024-03-26T20:52:42.718 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:42 smithi174 ceph-mon[26934]: osdmap e27: 4 total, 4 up, 4 in 2024-03-26T20:52:42.718 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:42 smithi174 ceph-mon[26934]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-26T20:52:42.718 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:42 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:52:42.755 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:42 smithi063 ceph-mon[24818]: osdmap e27: 4 total, 4 up, 4 in 2024-03-26T20:52:42.755 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:42 smithi063 ceph-mon[24818]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-26T20:52:42.755 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:42 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:52:44.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:44 smithi174 ceph-mon[26934]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-26T20:52:44.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:44 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-26T20:52:44.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:44 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-26T20:52:44.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:44 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:52:44.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:44 smithi063 ceph-mon[24818]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-26T20:52:44.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:44 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-26T20:52:44.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:44 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-26T20:52:44.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:44 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:52:45.619 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:45 smithi063 ceph-mon[24818]: from='client.24155 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi174:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:52:45.775 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:45 smithi174 ceph-mon[26934]: from='client.24155 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi174:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:52:47.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:46 smithi174 ceph-mon[26934]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-26T20:52:47.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:46 smithi174 ceph-mon[26934]: Detected new or changed devices on smithi063 2024-03-26T20:52:47.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:46 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:52:47.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:46 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-26T20:52:47.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:46 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-26T20:52:47.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:46 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-26T20:52:47.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:46 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-26T20:52:47.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:46 smithi174 ceph-mon[26934]: Adjusting osd_memory_target on smithi063 to 3216M 2024-03-26T20:52:47.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:46 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:52:47.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:46 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:52:47.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:46 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T20:52:47.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:46 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:52:47.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:46 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T20:52:47.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:46 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:52:47.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:46 smithi174 ceph-mon[26934]: from='client.? 172.21.15.174:0/1910084758' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d2574505-8824-4246-95ac-83d1d8a0b357"}]: dispatch 2024-03-26T20:52:47.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:46 smithi174 ceph-mon[26934]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d2574505-8824-4246-95ac-83d1d8a0b357"}]: dispatch 2024-03-26T20:52:47.050 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:46 smithi174 ceph-mon[26934]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d2574505-8824-4246-95ac-83d1d8a0b357"}]': finished 2024-03-26T20:52:47.050 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:46 smithi174 ceph-mon[26934]: osdmap e28: 5 total, 4 up, 5 in 2024-03-26T20:52:47.050 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:46 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-26T20:52:47.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:46 smithi063 ceph-mon[24818]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-26T20:52:47.052 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:46 smithi063 ceph-mon[24818]: Detected new or changed devices on smithi063 2024-03-26T20:52:47.052 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:46 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:52:47.052 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:46 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-26T20:52:47.052 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:46 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-26T20:52:47.052 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:46 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-26T20:52:47.052 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:46 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-26T20:52:47.052 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:46 smithi063 ceph-mon[24818]: Adjusting osd_memory_target on smithi063 to 3216M 2024-03-26T20:52:47.052 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:46 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:52:47.052 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:46 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:52:47.052 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:46 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T20:52:47.052 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:46 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:52:47.052 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:46 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T20:52:47.052 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:46 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:52:47.053 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:46 smithi063 ceph-mon[24818]: from='client.? 172.21.15.174:0/1910084758' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d2574505-8824-4246-95ac-83d1d8a0b357"}]: dispatch 2024-03-26T20:52:47.053 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:46 smithi063 ceph-mon[24818]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d2574505-8824-4246-95ac-83d1d8a0b357"}]: dispatch 2024-03-26T20:52:47.053 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:46 smithi063 ceph-mon[24818]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d2574505-8824-4246-95ac-83d1d8a0b357"}]': finished 2024-03-26T20:52:47.053 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:46 smithi063 ceph-mon[24818]: osdmap e28: 5 total, 4 up, 5 in 2024-03-26T20:52:47.053 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:46 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-26T20:52:48.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:47 smithi174 ceph-mon[26934]: from='client.? 172.21.15.174:0/158453702' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-26T20:52:48.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:47 smithi063 ceph-mon[24818]: from='client.? 172.21.15.174:0/158453702' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-26T20:52:48.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:48 smithi063 ceph-mon[24818]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-26T20:52:49.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:48 smithi174 ceph-mon[26934]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-26T20:52:50.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:50 smithi174 ceph-mon[26934]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-26T20:52:51.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:50 smithi063 ceph-mon[24818]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-26T20:52:52.988 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:52 smithi174 ceph-mon[26934]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-26T20:52:52.988 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:52 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-03-26T20:52:52.988 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:52 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:52:53.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:52 smithi063 ceph-mon[24818]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-26T20:52:53.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:52 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-03-26T20:52:53.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:52 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:52:53.953 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:53 smithi063 ceph-mon[24818]: Deploying daemon osd.4 on smithi174 2024-03-26T20:52:54.009 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:53 smithi174 ceph-mon[26934]: Deploying daemon osd.4 on smithi174 2024-03-26T20:52:54.969 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:54 smithi063 ceph-mon[24818]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-26T20:52:55.038 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:54 smithi174 ceph-mon[26934]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-26T20:52:56.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:56 smithi174 ceph-mon[26934]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-26T20:52:57.052 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:56 smithi063 ceph-mon[24818]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-26T20:52:58.538 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:58 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:52:58.538 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:58 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T20:52:58.538 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:58 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:52:58.538 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:58 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T20:52:58.538 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:52:58 smithi174 ceph-mon[26934]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-26T20:52:58.551 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:58 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:52:58.551 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:58 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T20:52:58.551 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:58 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:52:58.551 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:58 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T20:52:58.551 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:52:58 smithi063 ceph-mon[24818]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-26T20:52:59.920 INFO:teuthology.orchestra.run.smithi174.stdout:Created osd(s) 4 on host 'smithi174' 2024-03-26T20:53:00.888 DEBUG:teuthology.orchestra.run.smithi174:osd.4> sudo journalctl -f -n 0 -u ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715@osd.4.service 2024-03-26T20:53:00.890 INFO:tasks.cephadm:Deploying osd.5 on smithi174 with /dev/vg_nvme/lv_3... 2024-03-26T20:53:00.890 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-03-26T20:53:01.184 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:00 smithi174 ceph-mon[26934]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-26T20:53:01.184 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:00 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:53:01.184 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:00 smithi174 ceph-mon[26934]: from='osd.4 [v2:172.21.15.174:6800/2399757733,v1:172.21.15.174:6801/2399757733]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-26T20:53:01.184 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:00 smithi174 ceph-mon[26934]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-26T20:53:01.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:00 smithi063 ceph-mon[24818]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-26T20:53:01.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:00 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:53:01.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:00 smithi063 ceph-mon[24818]: from='osd.4 [v2:172.21.15.174:6800/2399757733,v1:172.21.15.174:6801/2399757733]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-26T20:53:01.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:00 smithi063 ceph-mon[24818]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-26T20:53:02.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:01 smithi174 ceph-mon[26934]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-03-26T20:53:02.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:01 smithi174 ceph-mon[26934]: osdmap e29: 5 total, 4 up, 5 in 2024-03-26T20:53:02.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:01 smithi174 ceph-mon[26934]: from='osd.4 [v2:172.21.15.174:6800/2399757733,v1:172.21.15.174:6801/2399757733]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]: dispatch 2024-03-26T20:53:02.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:01 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-26T20:53:02.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:01 smithi174 ceph-mon[26934]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]: dispatch 2024-03-26T20:53:02.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:01 smithi174 ceph-mon[26934]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-26T20:53:02.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:01 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:53:02.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:01 smithi174 ceph-mon[26934]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]': finished 2024-03-26T20:53:02.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:01 smithi174 ceph-mon[26934]: osdmap e30: 5 total, 4 up, 5 in 2024-03-26T20:53:02.299 INFO:journalctl@ceph.osd.4.smithi174.stdout:Mar 26 20:53:01 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-4[31184]: 2024-03-26T20:53:01.932+0000 7f3127989700 -1 osd.4 0 waiting for initial osdmap 2024-03-26T20:53:02.299 INFO:journalctl@ceph.osd.4.smithi174.stdout:Mar 26 20:53:01 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-4[31184]: 2024-03-26T20:53:01.940+0000 7f3122320700 -1 osd.4 30 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-26T20:53:02.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:01 smithi063 ceph-mon[24818]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-03-26T20:53:02.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:01 smithi063 ceph-mon[24818]: osdmap e29: 5 total, 4 up, 5 in 2024-03-26T20:53:02.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:01 smithi063 ceph-mon[24818]: from='osd.4 [v2:172.21.15.174:6800/2399757733,v1:172.21.15.174:6801/2399757733]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]: dispatch 2024-03-26T20:53:02.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:01 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-26T20:53:02.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:01 smithi063 ceph-mon[24818]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]: dispatch 2024-03-26T20:53:02.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:01 smithi063 ceph-mon[24818]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-03-26T20:53:02.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:01 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:53:02.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:01 smithi063 ceph-mon[24818]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]': finished 2024-03-26T20:53:02.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:01 smithi063 ceph-mon[24818]: osdmap e30: 5 total, 4 up, 5 in 2024-03-26T20:53:03.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:02 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-26T20:53:03.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:02 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-26T20:53:03.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:02 smithi174 ceph-mon[26934]: osd.4 [v2:172.21.15.174:6800/2399757733,v1:172.21.15.174:6801/2399757733] boot 2024-03-26T20:53:03.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:02 smithi174 ceph-mon[26934]: osdmap e31: 5 total, 5 up, 5 in 2024-03-26T20:53:03.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:02 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-26T20:53:03.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:02 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-26T20:53:03.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:02 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-26T20:53:03.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:02 smithi063 ceph-mon[24818]: osd.4 [v2:172.21.15.174:6800/2399757733,v1:172.21.15.174:6801/2399757733] boot 2024-03-26T20:53:03.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:02 smithi063 ceph-mon[24818]: osdmap e31: 5 total, 5 up, 5 in 2024-03-26T20:53:03.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:02 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-26T20:53:04.132 INFO:teuthology.orchestra.run.smithi174.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-03-26T20:53:04.132 INFO:teuthology.orchestra.run.smithi174.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-03-26T20:53:04.132 INFO:teuthology.orchestra.run.smithi174.stderr: stderr: 10+0 records in 2024-03-26T20:53:04.132 INFO:teuthology.orchestra.run.smithi174.stderr:10+0 records out 2024-03-26T20:53:04.133 INFO:teuthology.orchestra.run.smithi174.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0143155 s, 732 MB/s 2024-03-26T20:53:04.133 INFO:teuthology.orchestra.run.smithi174.stderr:--> Zapping successful for: 2024-03-26T20:53:04.410 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:04 smithi174 ceph-mon[26934]: purged_snaps scrub starts 2024-03-26T20:53:04.411 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:04 smithi174 ceph-mon[26934]: purged_snaps scrub ok 2024-03-26T20:53:04.411 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:04 smithi174 ceph-mon[26934]: osdmap e32: 5 total, 5 up, 5 in 2024-03-26T20:53:04.411 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:04 smithi174 ceph-mon[26934]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-03-26T20:53:04.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:04 smithi063 ceph-mon[24818]: purged_snaps scrub starts 2024-03-26T20:53:04.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:04 smithi063 ceph-mon[24818]: purged_snaps scrub ok 2024-03-26T20:53:04.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:04 smithi063 ceph-mon[24818]: osdmap e32: 5 total, 5 up, 5 in 2024-03-26T20:53:04.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:04 smithi063 ceph-mon[24818]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-03-26T20:53:04.955 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -- ceph orch daemon add osd smithi174:vg_nvme/lv_3 2024-03-26T20:53:05.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:05 smithi174 ceph-mon[26934]: osdmap e33: 5 total, 5 up, 5 in 2024-03-26T20:53:05.626 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:05 smithi063 ceph-mon[24818]: osdmap e33: 5 total, 5 up, 5 in 2024-03-26T20:53:06.538 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:06 smithi174 ceph-mon[26934]: osdmap e34: 5 total, 5 up, 5 in 2024-03-26T20:53:06.538 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:06 smithi174 ceph-mon[26934]: pgmap v111: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-03-26T20:53:06.551 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:06 smithi063 ceph-mon[24818]: osdmap e34: 5 total, 5 up, 5 in 2024-03-26T20:53:06.551 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:06 smithi063 ceph-mon[24818]: pgmap v111: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-03-26T20:53:07.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:07 smithi174 ceph-mon[26934]: osdmap e35: 5 total, 5 up, 5 in 2024-03-26T20:53:07.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:07 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:53:07.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:07 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-26T20:53:07.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:07 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:53:07.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:07 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:53:07.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:07 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T20:53:07.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:07 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:53:07.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:07 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T20:53:07.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:07 smithi063 ceph-mon[24818]: osdmap e35: 5 total, 5 up, 5 in 2024-03-26T20:53:07.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:07 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:53:07.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:07 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-26T20:53:07.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:07 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:53:07.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:07 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:53:07.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:07 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T20:53:07.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:07 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:53:07.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:07 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T20:53:08.775 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:08 smithi174 ceph-mon[26934]: Detected new or changed devices on smithi174 2024-03-26T20:53:08.775 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:08 smithi174 ceph-mon[26934]: Adjusting osd_memory_target on smithi174 to 15936M 2024-03-26T20:53:08.775 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:08 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:53:08.775 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:08 smithi174 ceph-mon[26934]: pgmap v113: 1 pgs: 1 peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail; 124 KiB/s, 0 objects/s recovering 2024-03-26T20:53:08.775 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:08 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi063.lxhujt/mirror_snapshot_schedule"}]: dispatch 2024-03-26T20:53:08.775 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:08 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi063.lxhujt/trash_purge_schedule"}]: dispatch 2024-03-26T20:53:08.775 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:08 smithi174 ceph-mon[26934]: from='client.24173 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi174:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:53:08.775 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:08 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-26T20:53:08.775 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:08 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-26T20:53:08.776 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:08 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:53:08.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:08 smithi063 ceph-mon[24818]: Detected new or changed devices on smithi174 2024-03-26T20:53:08.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:08 smithi063 ceph-mon[24818]: Adjusting osd_memory_target on smithi174 to 15936M 2024-03-26T20:53:08.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:08 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:53:08.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:08 smithi063 ceph-mon[24818]: pgmap v113: 1 pgs: 1 peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail; 124 KiB/s, 0 objects/s recovering 2024-03-26T20:53:08.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:08 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi063.lxhujt/mirror_snapshot_schedule"}]: dispatch 2024-03-26T20:53:08.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:08 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi063.lxhujt/trash_purge_schedule"}]: dispatch 2024-03-26T20:53:08.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:08 smithi063 ceph-mon[24818]: from='client.24173 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi174:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:53:08.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:08 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-26T20:53:08.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:08 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-26T20:53:08.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:08 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:53:10.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:10 smithi174 ceph-mon[26934]: pgmap v114: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-03-26T20:53:10.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:10 smithi174 ceph-mon[26934]: from='client.? 172.21.15.174:0/151591964' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b9bc8ccb-da0f-4bbc-88ef-ff861ee0892b"}]: dispatch 2024-03-26T20:53:10.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:10 smithi174 ceph-mon[26934]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b9bc8ccb-da0f-4bbc-88ef-ff861ee0892b"}]: dispatch 2024-03-26T20:53:10.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:10 smithi174 ceph-mon[26934]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b9bc8ccb-da0f-4bbc-88ef-ff861ee0892b"}]': finished 2024-03-26T20:53:10.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:10 smithi174 ceph-mon[26934]: osdmap e36: 6 total, 5 up, 6 in 2024-03-26T20:53:10.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:10 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-26T20:53:10.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:10 smithi063 ceph-mon[24818]: pgmap v114: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-03-26T20:53:10.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:10 smithi063 ceph-mon[24818]: from='client.? 172.21.15.174:0/151591964' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b9bc8ccb-da0f-4bbc-88ef-ff861ee0892b"}]: dispatch 2024-03-26T20:53:10.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:10 smithi063 ceph-mon[24818]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b9bc8ccb-da0f-4bbc-88ef-ff861ee0892b"}]: dispatch 2024-03-26T20:53:10.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:10 smithi063 ceph-mon[24818]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b9bc8ccb-da0f-4bbc-88ef-ff861ee0892b"}]': finished 2024-03-26T20:53:10.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:10 smithi063 ceph-mon[24818]: osdmap e36: 6 total, 5 up, 6 in 2024-03-26T20:53:10.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:10 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-26T20:53:11.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:11 smithi174 ceph-mon[26934]: from='client.? 172.21.15.174:0/2697944219' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-26T20:53:11.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:11 smithi063 ceph-mon[24818]: from='client.? 172.21.15.174:0/2697944219' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-26T20:53:12.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:12 smithi174 ceph-mon[26934]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 84 KiB/s, 0 objects/s recovering 2024-03-26T20:53:12.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:12 smithi063 ceph-mon[24818]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 84 KiB/s, 0 objects/s recovering 2024-03-26T20:53:14.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:14 smithi174 ceph-mon[26934]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-03-26T20:53:14.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:14 smithi063 ceph-mon[24818]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-03-26T20:53:16.670 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:16 smithi174 ceph-mon[26934]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-03-26T20:53:16.671 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:16 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-03-26T20:53:16.671 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:16 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:53:16.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:16 smithi063 ceph-mon[24818]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-03-26T20:53:16.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:16 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-03-26T20:53:16.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:16 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:53:17.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:17 smithi063 ceph-mon[24818]: Deploying daemon osd.5 on smithi174 2024-03-26T20:53:17.858 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:17 smithi174 ceph-mon[26934]: Deploying daemon osd.5 on smithi174 2024-03-26T20:53:18.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:18 smithi063 ceph-mon[24818]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-03-26T20:53:18.859 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:18 smithi174 ceph-mon[26934]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-03-26T20:53:20.731 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:20 smithi174 ceph-mon[26934]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-03-26T20:53:21.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:20 smithi063 ceph-mon[24818]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-03-26T20:53:22.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:22 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:53:22.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:22 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T20:53:22.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:22 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:53:22.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:22 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T20:53:22.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:22 smithi174 ceph-mon[26934]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-03-26T20:53:22.551 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:22 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:53:22.551 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:22 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T20:53:22.551 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:22 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:53:22.551 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:22 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T20:53:22.551 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:22 smithi063 ceph-mon[24818]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-03-26T20:53:24.130 INFO:teuthology.orchestra.run.smithi174.stdout:Created osd(s) 5 on host 'smithi174' 2024-03-26T20:53:24.787 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:24 smithi174 ceph-mon[26934]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-03-26T20:53:24.787 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:24 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:53:24.787 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:24 smithi174 ceph-mon[26934]: from='osd.5 [v2:172.21.15.174:6808/1022923234,v1:172.21.15.174:6809/1022923234]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-03-26T20:53:24.787 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:24 smithi174 ceph-mon[26934]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-03-26T20:53:24.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:24 smithi063 ceph-mon[24818]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-03-26T20:53:24.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:24 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:53:24.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:24 smithi063 ceph-mon[24818]: from='osd.5 [v2:172.21.15.174:6808/1022923234,v1:172.21.15.174:6809/1022923234]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-03-26T20:53:24.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:24 smithi063 ceph-mon[24818]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-03-26T20:53:25.130 DEBUG:teuthology.orchestra.run.smithi174:osd.5> sudo journalctl -f -n 0 -u ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715@osd.5.service 2024-03-26T20:53:25.132 INFO:tasks.cephadm:Deploying osd.6 on smithi174 with /dev/vg_nvme/lv_2... 2024-03-26T20:53:25.133 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-03-26T20:53:26.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:26 smithi174 ceph-mon[26934]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-03-26T20:53:26.621 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:26 smithi174 ceph-mon[26934]: osdmap e37: 6 total, 5 up, 6 in 2024-03-26T20:53:26.621 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:26 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-26T20:53:26.621 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:26 smithi174 ceph-mon[26934]: from='osd.5 [v2:172.21.15.174:6808/1022923234,v1:172.21.15.174:6809/1022923234]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]: dispatch 2024-03-26T20:53:26.621 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:26 smithi174 ceph-mon[26934]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]: dispatch 2024-03-26T20:53:26.621 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:26 smithi174 ceph-mon[26934]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-03-26T20:53:26.621 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 20:53:26 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-5[34148]: 2024-03-26T20:53:26.150+0000 7f435e388700 -1 osd.5 0 waiting for initial osdmap 2024-03-26T20:53:26.621 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 20:53:26 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-5[34148]: 2024-03-26T20:53:26.155+0000 7f435851e700 -1 osd.5 38 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-26T20:53:26.622 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:26 smithi063 ceph-mon[24818]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-03-26T20:53:26.622 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:26 smithi063 ceph-mon[24818]: osdmap e37: 6 total, 5 up, 6 in 2024-03-26T20:53:26.622 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:26 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-26T20:53:26.622 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:26 smithi063 ceph-mon[24818]: from='osd.5 [v2:172.21.15.174:6808/1022923234,v1:172.21.15.174:6809/1022923234]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]: dispatch 2024-03-26T20:53:26.622 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:26 smithi063 ceph-mon[24818]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]: dispatch 2024-03-26T20:53:26.622 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:26 smithi063 ceph-mon[24818]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-03-26T20:53:27.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:27 smithi174 ceph-mon[26934]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]': finished 2024-03-26T20:53:27.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:27 smithi174 ceph-mon[26934]: osdmap e38: 6 total, 5 up, 6 in 2024-03-26T20:53:27.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:27 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-26T20:53:27.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:27 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-26T20:53:27.554 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:27 smithi063 ceph-mon[24818]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]': finished 2024-03-26T20:53:27.554 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:27 smithi063 ceph-mon[24818]: osdmap e38: 6 total, 5 up, 6 in 2024-03-26T20:53:27.554 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:27 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-26T20:53:27.554 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:27 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-26T20:53:27.619 INFO:teuthology.orchestra.run.smithi174.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-03-26T20:53:27.620 INFO:teuthology.orchestra.run.smithi174.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-03-26T20:53:27.620 INFO:teuthology.orchestra.run.smithi174.stderr: stderr: 10+0 records in 2024-03-26T20:53:27.620 INFO:teuthology.orchestra.run.smithi174.stderr:10+0 records out 2024-03-26T20:53:27.620 INFO:teuthology.orchestra.run.smithi174.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0149806 s, 700 MB/s 2024-03-26T20:53:27.620 INFO:teuthology.orchestra.run.smithi174.stderr:--> Zapping successful for: 2024-03-26T20:53:28.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:28 smithi174 ceph-mon[26934]: purged_snaps scrub starts 2024-03-26T20:53:28.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:28 smithi174 ceph-mon[26934]: purged_snaps scrub ok 2024-03-26T20:53:28.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:28 smithi174 ceph-mon[26934]: osd.5 [v2:172.21.15.174:6808/1022923234,v1:172.21.15.174:6809/1022923234] boot 2024-03-26T20:53:28.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:28 smithi174 ceph-mon[26934]: osdmap e39: 6 total, 6 up, 6 in 2024-03-26T20:53:28.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:28 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-26T20:53:28.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:28 smithi174 ceph-mon[26934]: osdmap e40: 6 total, 6 up, 6 in 2024-03-26T20:53:28.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:28 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:53:28.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:28 smithi174 ceph-mon[26934]: pgmap v128: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-03-26T20:53:28.353 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -- ceph orch daemon add osd smithi174:vg_nvme/lv_2 2024-03-26T20:53:28.551 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:28 smithi063 ceph-mon[24818]: purged_snaps scrub starts 2024-03-26T20:53:28.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:28 smithi063 ceph-mon[24818]: purged_snaps scrub ok 2024-03-26T20:53:28.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:28 smithi063 ceph-mon[24818]: osd.5 [v2:172.21.15.174:6808/1022923234,v1:172.21.15.174:6809/1022923234] boot 2024-03-26T20:53:28.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:28 smithi063 ceph-mon[24818]: osdmap e39: 6 total, 6 up, 6 in 2024-03-26T20:53:28.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:28 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-26T20:53:28.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:28 smithi063 ceph-mon[24818]: osdmap e40: 6 total, 6 up, 6 in 2024-03-26T20:53:28.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:28 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:53:28.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:28 smithi063 ceph-mon[24818]: pgmap v128: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-03-26T20:53:29.359 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:29 smithi174 ceph-mon[26934]: osdmap e41: 6 total, 6 up, 6 in 2024-03-26T20:53:29.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:29 smithi063 ceph-mon[24818]: osdmap e41: 6 total, 6 up, 6 in 2024-03-26T20:53:30.551 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:30 smithi063 ceph-mon[24818]: pgmap v130: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail; 2/6 objects degraded (33.333%) 2024-03-26T20:53:30.617 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:30 smithi174 ceph-mon[26934]: pgmap v130: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail; 2/6 objects degraded (33.333%) 2024-03-26T20:53:31.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:31 smithi174 ceph-mon[26934]: Health check failed: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded (PG_DEGRADED) 2024-03-26T20:53:31.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:31 smithi174 ceph-mon[26934]: osdmap e42: 6 total, 6 up, 6 in 2024-03-26T20:53:31.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:31 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-26T20:53:31.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:31 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-26T20:53:31.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:31 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:53:31.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:31 smithi063 ceph-mon[24818]: Health check failed: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded (PG_DEGRADED) 2024-03-26T20:53:31.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:31 smithi063 ceph-mon[24818]: osdmap e42: 6 total, 6 up, 6 in 2024-03-26T20:53:31.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:31 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-26T20:53:31.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:31 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-26T20:53:31.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:31 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:53:32.445 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:32 smithi174 ceph-mon[26934]: from='client.24191 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi174:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:53:32.445 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:32 smithi174 ceph-mon[26934]: osdmap e43: 6 total, 6 up, 6 in 2024-03-26T20:53:32.445 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:32 smithi174 ceph-mon[26934]: pgmap v133: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail; 2/6 objects degraded (33.333%) 2024-03-26T20:53:32.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:32 smithi063 ceph-mon[24818]: from='client.24191 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi174:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:53:32.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:32 smithi063 ceph-mon[24818]: osdmap e43: 6 total, 6 up, 6 in 2024-03-26T20:53:32.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:32 smithi063 ceph-mon[24818]: pgmap v133: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail; 2/6 objects degraded (33.333%) 2024-03-26T20:53:34.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:34 smithi174 ceph-mon[26934]: Detected new or changed devices on smithi174 2024-03-26T20:53:34.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:34 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:53:34.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:34 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-26T20:53:34.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:34 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-26T20:53:34.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:34 smithi174 ceph-mon[26934]: Adjusting osd_memory_target on smithi174 to 7968M 2024-03-26T20:53:34.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:34 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:53:34.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:34 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:53:34.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:34 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T20:53:34.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:34 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:53:34.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:34 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T20:53:34.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:34 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:53:34.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:34 smithi174 ceph-mon[26934]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-03-26T20:53:34.551 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:34 smithi063 ceph-mon[24818]: Detected new or changed devices on smithi174 2024-03-26T20:53:34.551 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:34 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:53:34.551 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:34 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-26T20:53:34.551 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:34 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-26T20:53:34.551 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:34 smithi063 ceph-mon[24818]: Adjusting osd_memory_target on smithi174 to 7968M 2024-03-26T20:53:34.551 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:34 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:53:34.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:34 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:53:34.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:34 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T20:53:34.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:34 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:53:34.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:34 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T20:53:34.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:34 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:53:34.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:34 smithi063 ceph-mon[24818]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-03-26T20:53:35.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:35 smithi174 ceph-mon[26934]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded) 2024-03-26T20:53:35.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:35 smithi174 ceph-mon[26934]: Cluster is now healthy 2024-03-26T20:53:35.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:35 smithi174 ceph-mon[26934]: from='client.? 172.21.15.174:0/1031449279' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2283b8d5-fd69-467a-ad0d-b3b546d1494d"}]: dispatch 2024-03-26T20:53:35.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:35 smithi174 ceph-mon[26934]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2283b8d5-fd69-467a-ad0d-b3b546d1494d"}]: dispatch 2024-03-26T20:53:35.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:35 smithi174 ceph-mon[26934]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "2283b8d5-fd69-467a-ad0d-b3b546d1494d"}]': finished 2024-03-26T20:53:35.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:35 smithi174 ceph-mon[26934]: osdmap e44: 7 total, 6 up, 7 in 2024-03-26T20:53:35.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:35 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-26T20:53:35.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:35 smithi174 ceph-mon[26934]: from='client.? 172.21.15.174:0/912729778' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-26T20:53:35.554 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:35 smithi063 ceph-mon[24818]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded) 2024-03-26T20:53:35.554 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:35 smithi063 ceph-mon[24818]: Cluster is now healthy 2024-03-26T20:53:35.554 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:35 smithi063 ceph-mon[24818]: from='client.? 172.21.15.174:0/1031449279' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2283b8d5-fd69-467a-ad0d-b3b546d1494d"}]: dispatch 2024-03-26T20:53:35.554 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:35 smithi063 ceph-mon[24818]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2283b8d5-fd69-467a-ad0d-b3b546d1494d"}]: dispatch 2024-03-26T20:53:35.554 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:35 smithi063 ceph-mon[24818]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "2283b8d5-fd69-467a-ad0d-b3b546d1494d"}]': finished 2024-03-26T20:53:35.554 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:35 smithi063 ceph-mon[24818]: osdmap e44: 7 total, 6 up, 7 in 2024-03-26T20:53:35.554 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:35 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-26T20:53:35.554 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:35 smithi063 ceph-mon[24818]: from='client.? 172.21.15.174:0/912729778' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-26T20:53:36.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:36 smithi174 ceph-mon[26934]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-03-26T20:53:36.551 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:36 smithi063 ceph-mon[24818]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-03-26T20:53:38.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:38 smithi174 ceph-mon[26934]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-03-26T20:53:38.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:38 smithi063 ceph-mon[24818]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-03-26T20:53:40.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:40 smithi174 ceph-mon[26934]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 63 KiB/s, 0 objects/s recovering 2024-03-26T20:53:40.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:40 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-03-26T20:53:40.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:40 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:53:40.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:40 smithi063 ceph-mon[24818]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 63 KiB/s, 0 objects/s recovering 2024-03-26T20:53:40.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:40 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-03-26T20:53:40.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:40 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:53:41.775 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:41 smithi174 ceph-mon[26934]: Deploying daemon osd.6 on smithi174 2024-03-26T20:53:41.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:41 smithi063 ceph-mon[24818]: Deploying daemon osd.6 on smithi174 2024-03-26T20:53:42.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:42 smithi063 ceph-mon[24818]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-03-26T20:53:42.889 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:42 smithi174 ceph-mon[26934]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-03-26T20:53:44.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:44 smithi174 ceph-mon[26934]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-03-26T20:53:45.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:44 smithi063 ceph-mon[24818]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-03-26T20:53:46.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:46 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:53:46.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:46 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T20:53:46.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:46 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:53:46.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:46 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T20:53:46.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:46 smithi174 ceph-mon[26934]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-03-26T20:53:46.551 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:46 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:53:46.551 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:46 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T20:53:46.551 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:46 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:53:46.551 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:46 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T20:53:46.551 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:46 smithi063 ceph-mon[24818]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-03-26T20:53:48.532 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:48 smithi174 ceph-mon[26934]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-03-26T20:53:48.532 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:48 smithi174 ceph-mon[26934]: from='osd.6 [v2:172.21.15.174:6816/949715503,v1:172.21.15.174:6817/949715503]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-03-26T20:53:48.532 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:48 smithi174 ceph-mon[26934]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-03-26T20:53:48.541 INFO:teuthology.orchestra.run.smithi174.stdout:Created osd(s) 6 on host 'smithi174' 2024-03-26T20:53:48.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:48 smithi063 ceph-mon[24818]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-03-26T20:53:48.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:48 smithi063 ceph-mon[24818]: from='osd.6 [v2:172.21.15.174:6816/949715503,v1:172.21.15.174:6817/949715503]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-03-26T20:53:48.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:48 smithi063 ceph-mon[24818]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-03-26T20:53:49.244 DEBUG:teuthology.orchestra.run.smithi174:osd.6> sudo journalctl -f -n 0 -u ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715@osd.6.service 2024-03-26T20:53:49.247 INFO:tasks.cephadm:Deploying osd.7 on smithi174 with /dev/vg_nvme/lv_1... 2024-03-26T20:53:49.247 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-03-26T20:53:49.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:49 smithi174 ceph-mon[26934]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-03-26T20:53:49.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:49 smithi174 ceph-mon[26934]: osdmap e45: 7 total, 6 up, 7 in 2024-03-26T20:53:49.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:49 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-26T20:53:49.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:49 smithi174 ceph-mon[26934]: from='osd.6 [v2:172.21.15.174:6816/949715503,v1:172.21.15.174:6817/949715503]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]: dispatch 2024-03-26T20:53:49.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:49 smithi174 ceph-mon[26934]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]: dispatch 2024-03-26T20:53:49.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:49 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:53:49.549 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 20:53:49 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6[37195]: 2024-03-26T20:53:49.355+0000 7ff254114700 -1 osd.6 0 waiting for initial osdmap 2024-03-26T20:53:49.549 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 20:53:49 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6[37195]: 2024-03-26T20:53:49.361+0000 7ff250aaf700 -1 osd.6 46 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-26T20:53:49.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:49 smithi063 ceph-mon[24818]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-03-26T20:53:49.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:49 smithi063 ceph-mon[24818]: osdmap e45: 7 total, 6 up, 7 in 2024-03-26T20:53:49.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:49 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-26T20:53:49.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:49 smithi063 ceph-mon[24818]: from='osd.6 [v2:172.21.15.174:6816/949715503,v1:172.21.15.174:6817/949715503]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]: dispatch 2024-03-26T20:53:49.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:49 smithi063 ceph-mon[24818]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]: dispatch 2024-03-26T20:53:49.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:49 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:53:50.551 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:50 smithi063 ceph-mon[24818]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]': finished 2024-03-26T20:53:50.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:50 smithi063 ceph-mon[24818]: osdmap e46: 7 total, 6 up, 7 in 2024-03-26T20:53:50.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:50 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-26T20:53:50.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:50 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-26T20:53:50.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:50 smithi063 ceph-mon[24818]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-03-26T20:53:50.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:50 smithi063 ceph-mon[24818]: osd.6 [v2:172.21.15.174:6816/949715503,v1:172.21.15.174:6817/949715503] boot 2024-03-26T20:53:50.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:50 smithi063 ceph-mon[24818]: osdmap e47: 7 total, 7 up, 7 in 2024-03-26T20:53:50.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:50 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-26T20:53:50.651 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:50 smithi174 ceph-mon[26934]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]': finished 2024-03-26T20:53:50.651 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:50 smithi174 ceph-mon[26934]: osdmap e46: 7 total, 6 up, 7 in 2024-03-26T20:53:50.651 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:50 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-26T20:53:50.651 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:50 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-26T20:53:50.652 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:50 smithi174 ceph-mon[26934]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-03-26T20:53:50.652 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:50 smithi174 ceph-mon[26934]: osd.6 [v2:172.21.15.174:6816/949715503,v1:172.21.15.174:6817/949715503] boot 2024-03-26T20:53:50.652 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:50 smithi174 ceph-mon[26934]: osdmap e47: 7 total, 7 up, 7 in 2024-03-26T20:53:50.652 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:50 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-26T20:53:51.376 INFO:teuthology.orchestra.run.smithi174.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-03-26T20:53:51.376 INFO:teuthology.orchestra.run.smithi174.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-03-26T20:53:51.376 INFO:teuthology.orchestra.run.smithi174.stderr: stderr: 10+0 records in 2024-03-26T20:53:51.376 INFO:teuthology.orchestra.run.smithi174.stderr:10+0 records out 2024-03-26T20:53:51.376 INFO:teuthology.orchestra.run.smithi174.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0127165 s, 825 MB/s 2024-03-26T20:53:51.376 INFO:teuthology.orchestra.run.smithi174.stderr:--> Zapping successful for: 2024-03-26T20:53:52.111 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -- ceph orch daemon add osd smithi174:vg_nvme/lv_1 2024-03-26T20:53:52.414 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:52 smithi174 ceph-mon[26934]: purged_snaps scrub starts 2024-03-26T20:53:52.415 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:52 smithi174 ceph-mon[26934]: purged_snaps scrub ok 2024-03-26T20:53:52.415 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:52 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:53:52.415 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:52 smithi174 ceph-mon[26934]: osdmap e48: 7 total, 7 up, 7 in 2024-03-26T20:53:52.415 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:52 smithi174 ceph-mon[26934]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-03-26T20:53:52.551 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:52 smithi063 ceph-mon[24818]: purged_snaps scrub starts 2024-03-26T20:53:52.551 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:52 smithi063 ceph-mon[24818]: purged_snaps scrub ok 2024-03-26T20:53:52.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:52 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:53:52.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:52 smithi063 ceph-mon[24818]: osdmap e48: 7 total, 7 up, 7 in 2024-03-26T20:53:52.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:52 smithi063 ceph-mon[24818]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-03-26T20:53:53.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:53 smithi174 ceph-mon[26934]: osdmap e49: 7 total, 7 up, 7 in 2024-03-26T20:53:53.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:53 smithi063 ceph-mon[24818]: osdmap e49: 7 total, 7 up, 7 in 2024-03-26T20:53:54.642 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:54 smithi174 ceph-mon[26934]: osdmap e50: 7 total, 7 up, 7 in 2024-03-26T20:53:54.643 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:54 smithi174 ceph-mon[26934]: pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-03-26T20:53:54.702 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:54 smithi063 ceph-mon[24818]: osdmap e50: 7 total, 7 up, 7 in 2024-03-26T20:53:54.703 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:54 smithi063 ceph-mon[24818]: pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-03-26T20:53:55.741 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:55 smithi174 ceph-mon[26934]: osdmap e51: 7 total, 7 up, 7 in 2024-03-26T20:53:55.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:55 smithi063 ceph-mon[24818]: osdmap e51: 7 total, 7 up, 7 in 2024-03-26T20:53:56.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:56 smithi174 ceph-mon[26934]: from='client.24209 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi174:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:53:56.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:56 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-26T20:53:56.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:56 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-26T20:53:56.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:56 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:53:56.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:56 smithi174 ceph-mon[26934]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 124 KiB/s, 0 objects/s recovering 2024-03-26T20:53:56.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:56 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:53:56.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:56 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-26T20:53:56.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:56 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-26T20:53:56.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:56 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-26T20:53:56.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:56 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:53:56.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:56 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:53:56.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:56 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T20:53:56.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:56 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:53:56.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:56 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T20:53:56.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:56 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:53:56.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:56 smithi063 ceph-mon[24818]: from='client.24209 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi174:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:53:56.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:56 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-26T20:53:56.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:56 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-26T20:53:56.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:56 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:53:56.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:56 smithi063 ceph-mon[24818]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 124 KiB/s, 0 objects/s recovering 2024-03-26T20:53:56.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:56 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:53:56.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:56 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-26T20:53:56.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:56 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-26T20:53:56.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:56 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-26T20:53:56.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:56 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:53:56.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:56 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:53:56.803 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:56 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T20:53:56.803 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:56 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:53:56.803 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:56 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T20:53:56.803 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:56 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:53:57.425 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:57 smithi174 ceph-mon[26934]: Detected new or changed devices on smithi174 2024-03-26T20:53:57.425 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:57 smithi174 ceph-mon[26934]: Adjusting osd_memory_target on smithi174 to 5312M 2024-03-26T20:53:57.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:57 smithi063 ceph-mon[24818]: Detected new or changed devices on smithi174 2024-03-26T20:53:57.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:57 smithi063 ceph-mon[24818]: Adjusting osd_memory_target on smithi174 to 5312M 2024-03-26T20:53:58.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:58 smithi174 ceph-mon[26934]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-03-26T20:53:58.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:58 smithi174 ceph-mon[26934]: from='client.? 172.21.15.174:0/220214470' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "adf5696d-40cb-4701-8766-738480428100"}]: dispatch 2024-03-26T20:53:58.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:58 smithi174 ceph-mon[26934]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "adf5696d-40cb-4701-8766-738480428100"}]: dispatch 2024-03-26T20:53:58.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:58 smithi174 ceph-mon[26934]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "adf5696d-40cb-4701-8766-738480428100"}]': finished 2024-03-26T20:53:58.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:58 smithi174 ceph-mon[26934]: osdmap e52: 8 total, 7 up, 8 in 2024-03-26T20:53:58.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:58 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-26T20:53:58.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:58 smithi063 ceph-mon[24818]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-03-26T20:53:58.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:58 smithi063 ceph-mon[24818]: from='client.? 172.21.15.174:0/220214470' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "adf5696d-40cb-4701-8766-738480428100"}]: dispatch 2024-03-26T20:53:58.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:58 smithi063 ceph-mon[24818]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "adf5696d-40cb-4701-8766-738480428100"}]: dispatch 2024-03-26T20:53:58.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:58 smithi063 ceph-mon[24818]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "adf5696d-40cb-4701-8766-738480428100"}]': finished 2024-03-26T20:53:58.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:58 smithi063 ceph-mon[24818]: osdmap e52: 8 total, 7 up, 8 in 2024-03-26T20:53:58.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:58 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-26T20:53:59.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:53:59 smithi174 ceph-mon[26934]: from='client.? 172.21.15.174:0/3261699441' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-26T20:53:59.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:53:59 smithi063 ceph-mon[24818]: from='client.? 172.21.15.174:0/3261699441' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-26T20:54:00.551 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:00 smithi063 ceph-mon[24818]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 84 KiB/s, 0 objects/s recovering 2024-03-26T20:54:00.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:00 smithi174 ceph-mon[26934]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 84 KiB/s, 0 objects/s recovering 2024-03-26T20:54:02.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:02 smithi063 ceph-mon[24818]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-03-26T20:54:03.019 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:02 smithi174 ceph-mon[26934]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-03-26T20:54:04.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:04 smithi063 ceph-mon[24818]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 57 KiB/s, 0 objects/s recovering 2024-03-26T20:54:04.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:04 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-03-26T20:54:04.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:04 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:54:04.908 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:04 smithi174 ceph-mon[26934]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 57 KiB/s, 0 objects/s recovering 2024-03-26T20:54:04.908 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:04 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-03-26T20:54:04.909 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:04 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:54:05.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:05 smithi174 ceph-mon[26934]: Deploying daemon osd.7 on smithi174 2024-03-26T20:54:05.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:05 smithi063 ceph-mon[24818]: Deploying daemon osd.7 on smithi174 2024-03-26T20:54:06.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:06 smithi174 ceph-mon[26934]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-03-26T20:54:07.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:06 smithi063 ceph-mon[24818]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-03-26T20:54:08.444 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:08 smithi174 ceph-mon[26934]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-03-26T20:54:08.445 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:08 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi063.lxhujt/mirror_snapshot_schedule"}]: dispatch 2024-03-26T20:54:08.445 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:08 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi063.lxhujt/trash_purge_schedule"}]: dispatch 2024-03-26T20:54:08.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:08 smithi063 ceph-mon[24818]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-03-26T20:54:08.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:08 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi063.lxhujt/mirror_snapshot_schedule"}]: dispatch 2024-03-26T20:54:08.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:08 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi063.lxhujt/trash_purge_schedule"}]: dispatch 2024-03-26T20:54:10.897 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:10 smithi174 ceph-mon[26934]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-03-26T20:54:10.898 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:10 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:54:10.898 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:10 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T20:54:10.898 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:10 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:54:10.898 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:10 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T20:54:11.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:10 smithi063 ceph-mon[24818]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-03-26T20:54:11.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:10 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:54:11.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:10 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T20:54:11.052 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:10 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:54:11.052 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:10 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T20:54:12.975 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:12 smithi174 ceph-mon[26934]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-03-26T20:54:13.006 INFO:teuthology.orchestra.run.smithi174.stdout:Created osd(s) 7 on host 'smithi174' 2024-03-26T20:54:13.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:12 smithi063 ceph-mon[24818]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-03-26T20:54:13.687 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:13 smithi174 ceph-mon[26934]: from='osd.7 [v2:172.21.15.174:6824/2501302609,v1:172.21.15.174:6825/2501302609]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-03-26T20:54:13.687 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:13 smithi174 ceph-mon[26934]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-03-26T20:54:13.688 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:13 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:54:13.714 DEBUG:teuthology.orchestra.run.smithi174:osd.7> sudo journalctl -f -n 0 -u ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715@osd.7.service 2024-03-26T20:54:13.716 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-03-26T20:54:13.717 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -- ceph osd stat -f json 2024-03-26T20:54:14.037 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:13 smithi063 ceph-mon[24818]: from='osd.7 [v2:172.21.15.174:6824/2501302609,v1:172.21.15.174:6825/2501302609]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-03-26T20:54:14.037 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:13 smithi063 ceph-mon[24818]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-03-26T20:54:14.038 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:13 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:54:14.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:14 smithi063 ceph-mon[24818]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-03-26T20:54:14.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:14 smithi063 ceph-mon[24818]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-03-26T20:54:14.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:14 smithi063 ceph-mon[24818]: osdmap e53: 8 total, 7 up, 8 in 2024-03-26T20:54:14.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:14 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-26T20:54:14.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:14 smithi063 ceph-mon[24818]: from='osd.7 [v2:172.21.15.174:6824/2501302609,v1:172.21.15.174:6825/2501302609]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]: dispatch 2024-03-26T20:54:14.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:14 smithi063 ceph-mon[24818]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]: dispatch 2024-03-26T20:54:14.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:14 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:54:14.934 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:14 smithi174 ceph-mon[26934]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-03-26T20:54:14.934 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:14 smithi174 ceph-mon[26934]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-03-26T20:54:14.934 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:14 smithi174 ceph-mon[26934]: osdmap e53: 8 total, 7 up, 8 in 2024-03-26T20:54:14.934 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:14 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-26T20:54:14.934 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:14 smithi174 ceph-mon[26934]: from='osd.7 [v2:172.21.15.174:6824/2501302609,v1:172.21.15.174:6825/2501302609]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]: dispatch 2024-03-26T20:54:14.934 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:14 smithi174 ceph-mon[26934]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]: dispatch 2024-03-26T20:54:14.934 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:14 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:54:14.934 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 20:54:14 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[40194]: 2024-03-26T20:54:14.600+0000 7f54b8b25700 -1 osd.7 0 waiting for initial osdmap 2024-03-26T20:54:14.935 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 20:54:14 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[40194]: 2024-03-26T20:54:14.607+0000 7f54b24ba700 -1 osd.7 54 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-26T20:54:15.767 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-03-26T20:54:15.767 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:15 smithi063 ceph-mon[24818]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]': finished 2024-03-26T20:54:15.768 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:15 smithi063 ceph-mon[24818]: osdmap e54: 8 total, 7 up, 8 in 2024-03-26T20:54:15.768 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:15 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-26T20:54:15.768 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:15 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-26T20:54:15.895 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:15 smithi174 ceph-mon[26934]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]': finished 2024-03-26T20:54:15.895 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:15 smithi174 ceph-mon[26934]: osdmap e54: 8 total, 7 up, 8 in 2024-03-26T20:54:15.895 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:15 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-26T20:54:15.896 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:15 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-26T20:54:16.546 INFO:teuthology.orchestra.run.smithi063.stdout:{"epoch":55,"num_osds":8,"num_up_osds":8,"osd_up_since":1711486455,"num_in_osds":8,"osd_in_since":1711486438,"num_remapped_pgs":1} 2024-03-26T20:54:16.546 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -- ceph osd dump --format=json 2024-03-26T20:54:16.778 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/mon.smithi063/config 2024-03-26T20:54:16.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:16 smithi063 ceph-mon[24818]: purged_snaps scrub starts 2024-03-26T20:54:16.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:16 smithi063 ceph-mon[24818]: purged_snaps scrub ok 2024-03-26T20:54:16.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:16 smithi063 ceph-mon[24818]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-03-26T20:54:16.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:16 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-26T20:54:16.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:16 smithi063 ceph-mon[24818]: osd.7 [v2:172.21.15.174:6824/2501302609,v1:172.21.15.174:6825/2501302609] boot 2024-03-26T20:54:16.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:16 smithi063 ceph-mon[24818]: osdmap e55: 8 total, 8 up, 8 in 2024-03-26T20:54:16.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:16 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-26T20:54:16.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:16 smithi063 ceph-mon[24818]: from='client.? 172.21.15.63:0/2894054229' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-03-26T20:54:16.968 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:16 smithi174 ceph-mon[26934]: purged_snaps scrub starts 2024-03-26T20:54:16.969 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:16 smithi174 ceph-mon[26934]: purged_snaps scrub ok 2024-03-26T20:54:16.969 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:16 smithi174 ceph-mon[26934]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-03-26T20:54:16.969 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:16 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-26T20:54:16.969 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:16 smithi174 ceph-mon[26934]: osd.7 [v2:172.21.15.174:6824/2501302609,v1:172.21.15.174:6825/2501302609] boot 2024-03-26T20:54:16.969 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:16 smithi174 ceph-mon[26934]: osdmap e55: 8 total, 8 up, 8 in 2024-03-26T20:54:16.969 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:16 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-26T20:54:16.969 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:16 smithi174 ceph-mon[26934]: from='client.? 172.21.15.63:0/2894054229' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-03-26T20:54:17.897 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:17 smithi174 ceph-mon[26934]: osdmap e56: 8 total, 8 up, 8 in 2024-03-26T20:54:17.897 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:17 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:54:17.897 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:17 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-26T20:54:17.897 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:17 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-26T20:54:17.897 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:17 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-26T20:54:17.897 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:17 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-26T20:54:17.897 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:17 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:54:17.897 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:17 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:54:17.898 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:17 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T20:54:17.898 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:17 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:54:17.898 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:17 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T20:54:17.898 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:17 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:54:17.941 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:17 smithi063 ceph-mon[24818]: osdmap e56: 8 total, 8 up, 8 in 2024-03-26T20:54:17.942 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:17 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:54:17.942 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:17 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-26T20:54:17.942 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:17 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-26T20:54:17.942 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:17 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-26T20:54:17.942 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:17 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-26T20:54:17.942 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:17 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:54:17.942 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:17 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:54:17.942 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:17 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T20:54:17.942 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:17 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:54:17.942 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:17 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T20:54:17.942 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:17 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:54:18.566 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-03-26T20:54:18.567 INFO:teuthology.orchestra.run.smithi063.stdout:{"epoch":57,"fsid":"0ed4dd4c-ebb2-11ee-95d0-87774f69a715","created":"2024-03-26T20:48:14.448236+0000","modified":"2024-03-26T20:54:17.605492+0000","last_up_change":"2024-03-26T20:54:15.599230+0000","last_in_change":"2024-03-26T20:53:58.130151+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-26T20:52:13.796312+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"6b0f79e5-c540-4701-a03e-38b2596b333b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6802","nonce":2118194723},{"type":"v1","addr":"172.21.15.63:6803","nonce":2118194723}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6804","nonce":2118194723},{"type":"v1","addr":"172.21.15.63:6805","nonce":2118194723}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6808","nonce":2118194723},{"type":"v1","addr":"172.21.15.63:6809","nonce":2118194723}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6806","nonce":2118194723},{"type":"v1","addr":"172.21.15.63:6807","nonce":2118194723}]},"public_addr":"172.21.15.63:6803/2118194723","cluster_addr":"172.21.15.63:6805/2118194723","heartbeat_back_addr":"172.21.15.63:6809/2118194723","heartbeat_front_addr":"172.21.15.63:6807/2118194723","state":["exists","up"]},{"osd":1,"uuid":"cc56c3cf-5bdc-4d3b-b1fc-57f20f9c9509","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6810","nonce":4265583190},{"type":"v1","addr":"172.21.15.63:6811","nonce":4265583190}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6812","nonce":4265583190},{"type":"v1","addr":"172.21.15.63:6813","nonce":4265583190}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6816","nonce":4265583190},{"type":"v1","addr":"172.21.15.63:6817","nonce":4265583190}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6814","nonce":4265583190},{"type":"v1","addr":"172.21.15.63:6815","nonce":4265583190}]},"public_addr":"172.21.15.63:6811/4265583190","cluster_addr":"172.21.15.63:6813/4265583190","heartbeat_back_addr":"172.21.15.63:6817/4265583190","heartbeat_front_addr":"172.21.15.63:6815/4265583190","state":["exists","up"]},{"osd":2,"uuid":"5dcc41b7-668e-46c4-a389-e8a6cf2a5f64","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6818","nonce":2962020712},{"type":"v1","addr":"172.21.15.63:6819","nonce":2962020712}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6820","nonce":2962020712},{"type":"v1","addr":"172.21.15.63:6821","nonce":2962020712}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6824","nonce":2962020712},{"type":"v1","addr":"172.21.15.63:6825","nonce":2962020712}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6822","nonce":2962020712},{"type":"v1","addr":"172.21.15.63:6823","nonce":2962020712}]},"public_addr":"172.21.15.63:6819/2962020712","cluster_addr":"172.21.15.63:6821/2962020712","heartbeat_back_addr":"172.21.15.63:6825/2962020712","heartbeat_front_addr":"172.21.15.63:6823/2962020712","state":["exists","up"]},{"osd":3,"uuid":"21eb7156-f9a5-4108-982d-47847322b16e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6826","nonce":284512069},{"type":"v1","addr":"172.21.15.63:6827","nonce":284512069}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6828","nonce":284512069},{"type":"v1","addr":"172.21.15.63:6829","nonce":284512069}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6832","nonce":284512069},{"type":"v1","addr":"172.21.15.63:6833","nonce":284512069}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6830","nonce":284512069},{"type":"v1","addr":"172.21.15.63:6831","nonce":284512069}]},"public_addr":"172.21.15.63:6827/284512069","cluster_addr":"172.21.15.63:6829/284512069","heartbeat_back_addr":"172.21.15.63:6833/284512069","heartbeat_front_addr":"172.21.15.63:6831/284512069","state":["exists","up"]},{"osd":4,"uuid":"d2574505-8824-4246-95ac-83d1d8a0b357","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6800","nonce":2399757733},{"type":"v1","addr":"172.21.15.174:6801","nonce":2399757733}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6802","nonce":2399757733},{"type":"v1","addr":"172.21.15.174:6803","nonce":2399757733}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6806","nonce":2399757733},{"type":"v1","addr":"172.21.15.174:6807","nonce":2399757733}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6804","nonce":2399757733},{"type":"v1","addr":"172.21.15.174:6805","nonce":2399757733}]},"public_addr":"172.21.15.174:6801/2399757733","cluster_addr":"172.21.15.174:6803/2399757733","heartbeat_back_addr":"172.21.15.174:6807/2399757733","heartbeat_front_addr":"172.21.15.174:6805/2399757733","state":["exists","up"]},{"osd":5,"uuid":"b9bc8ccb-da0f-4bbc-88ef-ff861ee0892b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6808","nonce":1022923234},{"type":"v1","addr":"172.21.15.174:6809","nonce":1022923234}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6810","nonce":1022923234},{"type":"v1","addr":"172.21.15.174:6811","nonce":1022923234}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6814","nonce":1022923234},{"type":"v1","addr":"172.21.15.174:6815","nonce":1022923234}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6812","nonce":1022923234},{"type":"v1","addr":"172.21.15.174:6813","nonce":1022923234}]},"public_addr":"172.21.15.174:6809/1022923234","cluster_addr":"172.21.15.174:6811/1022923234","heartbeat_back_addr":"172.21.15.174:6815/1022923234","heartbeat_front_addr":"172.21.15.174:6813/1022923234","state":["exists","up"]},{"osd":6,"uuid":"2283b8d5-fd69-467a-ad0d-b3b546d1494d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6816","nonce":949715503},{"type":"v1","addr":"172.21.15.174:6817","nonce":949715503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6818","nonce":949715503},{"type":"v1","addr":"172.21.15.174:6819","nonce":949715503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6822","nonce":949715503},{"type":"v1","addr":"172.21.15.174:6823","nonce":949715503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6820","nonce":949715503},{"type":"v1","addr":"172.21.15.174:6821","nonce":949715503}]},"public_addr":"172.21.15.174:6817/949715503","cluster_addr":"172.21.15.174:6819/949715503","heartbeat_back_addr":"172.21.15.174:6823/949715503","heartbeat_front_addr":"172.21.15.174:6821/949715503","state":["exists","up"]},{"osd":7,"uuid":"adf5696d-40cb-4701-8766-738480428100","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6824","nonce":2501302609},{"type":"v1","addr":"172.21.15.174:6825","nonce":2501302609}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6826","nonce":2501302609},{"type":"v1","addr":"172.21.15.174:6827","nonce":2501302609}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6830","nonce":2501302609},{"type":"v1","addr":"172.21.15.174:6831","nonce":2501302609}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6828","nonce":2501302609},{"type":"v1","addr":"172.21.15.174:6829","nonce":2501302609}]},"public_addr":"172.21.15.174:6825/2501302609","cluster_addr":"172.21.15.174:6827/2501302609","heartbeat_back_addr":"172.21.15.174:6831/2501302609","heartbeat_front_addr":"172.21.15.174:6829/2501302609","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-26T20:51:11.316361+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-26T20:51:43.245122+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-26T20:52:11.736302+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-26T20:52:39.294718+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-26T20:53:01.346252+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-26T20:53:25.308475+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-26T20:53:49.290005+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-26T20:54:13.664236+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[{"pgid":"1.0","osds":[0,6]}],"primary_temp":[],"blocklist":{"172.21.15.63:0/2441187239":"2024-03-27T20:50:06.915397+0000","172.21.15.63:6800/2036968009":"2024-03-27T20:50:06.915397+0000","172.21.15.63:0/1102163987":"2024-03-27T20:50:06.915397+0000","172.21.15.63:0/3588541937":"2024-03-27T20:49:12.588686+0000","172.21.15.63:6801/2789315162":"2024-03-27T20:49:12.588686+0000","172.21.15.63:0/3031931285":"2024-03-27T20:50:06.915397+0000","172.21.15.63:6800/2789315162":"2024-03-27T20:49:12.588686+0000","172.21.15.63:0/2089226072":"2024-03-27T20:49:12.588686+0000","172.21.15.63:6801/2036968009":"2024-03-27T20:50:06.915397+0000","172.21.15.63:6800/3345547800":"2024-03-27T20:48:42.768729+0000","172.21.15.63:6801/3345547800":"2024-03-27T20:48:42.768729+0000","172.21.15.63:0/3861012011":"2024-03-27T20:48:42.768729+0000","172.21.15.63:0/798866617":"2024-03-27T20:49:12.588686+0000","172.21.15.63:0/1772414794":"2024-03-27T20:48:42.768729+0000","172.21.15.63:0/1448834781":"2024-03-27T20:48:42.768729+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-26T20:54:19.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:18 smithi174 ceph-mon[26934]: Detected new or changed devices on smithi174 2024-03-26T20:54:19.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:18 smithi174 ceph-mon[26934]: Adjusting osd_memory_target on smithi174 to 3984M 2024-03-26T20:54:19.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:18 smithi174 ceph-mon[26934]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-03-26T20:54:19.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:18 smithi174 ceph-mon[26934]: osdmap e57: 8 total, 8 up, 8 in 2024-03-26T20:54:19.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:18 smithi174 ceph-mon[26934]: from='client.? 172.21.15.63:0/2136374753' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-26T20:54:19.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:18 smithi063 ceph-mon[24818]: Detected new or changed devices on smithi174 2024-03-26T20:54:19.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:18 smithi063 ceph-mon[24818]: Adjusting osd_memory_target on smithi174 to 3984M 2024-03-26T20:54:19.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:18 smithi063 ceph-mon[24818]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-03-26T20:54:19.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:18 smithi063 ceph-mon[24818]: osdmap e57: 8 total, 8 up, 8 in 2024-03-26T20:54:19.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:18 smithi063 ceph-mon[24818]: from='client.? 172.21.15.63:0/2136374753' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-26T20:54:19.266 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-26T20:52:13.796312+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-03-26T20:54:19.266 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -- ceph osd pool get .mgr pg_num 2024-03-26T20:54:19.488 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/mon.smithi063/config 2024-03-26T20:54:19.977 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:19 smithi063 ceph-mon[24818]: osdmap e58: 8 total, 8 up, 8 in 2024-03-26T20:54:20.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:19 smithi174 ceph-mon[26934]: osdmap e58: 8 total, 8 up, 8 in 2024-03-26T20:54:20.914 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:20 smithi063 ceph-mon[24818]: pgmap v172: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-03-26T20:54:20.914 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:20 smithi063 ceph-mon[24818]: osdmap e59: 8 total, 8 up, 8 in 2024-03-26T20:54:21.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:20 smithi174 ceph-mon[26934]: pgmap v172: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-03-26T20:54:21.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:20 smithi174 ceph-mon[26934]: osdmap e59: 8 total, 8 up, 8 in 2024-03-26T20:54:21.302 INFO:teuthology.orchestra.run.smithi063.stdout:pg_num: 1 2024-03-26T20:54:21.962 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:21 smithi063 ceph-mon[24818]: from='client.? 172.21.15.63:0/3929670631' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-03-26T20:54:21.988 INFO:tasks.cephadm:Setting up client nodes... 2024-03-26T20:54:21.988 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-03-26T20:54:22.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:21 smithi174 ceph-mon[26934]: from='client.? 172.21.15.63:0/3929670631' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-03-26T20:54:23.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:22 smithi174 ceph-mon[26934]: pgmap v174: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 105 KiB/s, 0 objects/s recovering 2024-03-26T20:54:23.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:22 smithi063 ceph-mon[24818]: pgmap v174: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 105 KiB/s, 0 objects/s recovering 2024-03-26T20:54:24.109 INFO:teuthology.orchestra.run.smithi063.stdout:[client.0] 2024-03-26T20:54:24.109 INFO:teuthology.orchestra.run.smithi063.stdout: key = AQAANgNmahqpBhAAJCv/vsbVS5zyUT0WrXmxYg== 2024-03-26T20:54:24.793 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:24 smithi063 ceph-mon[24818]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-03-26T20:54:24.794 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:24 smithi063 ceph-mon[24818]: from='client.? 172.21.15.63:0/799692475' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-03-26T20:54:24.794 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:24 smithi063 ceph-mon[24818]: from='client.? 172.21.15.63:0/799692475' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-03-26T20:54:24.818 DEBUG:teuthology.orchestra.run.smithi063:> set -ex 2024-03-26T20:54:24.818 DEBUG:teuthology.orchestra.run.smithi063:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-03-26T20:54:24.818 DEBUG:teuthology.orchestra.run.smithi063:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-03-26T20:54:24.863 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-03-26T20:54:24.863 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-03-26T20:54:24.863 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -- ceph mgr dump --format=json 2024-03-26T20:54:25.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:24 smithi174 ceph-mon[26934]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-03-26T20:54:25.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:24 smithi174 ceph-mon[26934]: from='client.? 172.21.15.63:0/799692475' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-03-26T20:54:25.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:24 smithi174 ceph-mon[26934]: from='client.? 172.21.15.63:0/799692475' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-03-26T20:54:25.120 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/mon.smithi063/config 2024-03-26T20:54:27.025 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-03-26T20:54:27.025 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:26 smithi063 ceph-mon[24818]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-03-26T20:54:27.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:26 smithi174 ceph-mon[26934]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-03-26T20:54:27.815 INFO:teuthology.orchestra.run.smithi063.stdout:{"epoch":19,"active_gid":14204,"active_name":"smithi063.lxhujt","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6800","nonce":2363950533},{"type":"v1","addr":"172.21.15.63:6801","nonce":2363950533}]},"active_addr":"172.21.15.63:6801/2363950533","active_change":"2024-03-26T20:50:06.915720+0000","active_mgr_features":4540138303579357183,"available":true,"standbys":[{"gid":14222,"name":"smithi174.sexoin","mgr_features":4540138303579357183,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.63:8443/","prometheus":"http://172.21.15.63:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"last_failure_osd_epoch":5,"active_clients":[{"addrvec":[{"type":"v2","addr":"172.21.15.63:0","nonce":2249244737}]},{"addrvec":[{"type":"v2","addr":"172.21.15.63:0","nonce":3804590783}]},{"addrvec":[{"type":"v2","addr":"172.21.15.63:0","nonce":2038086306}]},{"addrvec":[{"type":"v2","addr":"172.21.15.63:0","nonce":3453961544}]}]}} 2024-03-26T20:54:27.821 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-03-26T20:54:27.821 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-03-26T20:54:27.822 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -- ceph osd dump --format=json 2024-03-26T20:54:28.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:27 smithi174 ceph-mon[26934]: from='client.? 172.21.15.63:0/2460786911' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-03-26T20:54:28.100 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/mon.smithi063/config 2024-03-26T20:54:28.100 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:27 smithi063 ceph-mon[24818]: from='client.? 172.21.15.63:0/2460786911' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-03-26T20:54:28.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:28 smithi063 ceph-mon[24818]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-26T20:54:29.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:28 smithi174 ceph-mon[26934]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-26T20:54:29.865 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-03-26T20:54:29.865 INFO:teuthology.orchestra.run.smithi063.stdout:{"epoch":59,"fsid":"0ed4dd4c-ebb2-11ee-95d0-87774f69a715","created":"2024-03-26T20:48:14.448236+0000","modified":"2024-03-26T20:54:19.615732+0000","last_up_change":"2024-03-26T20:54:15.599230+0000","last_in_change":"2024-03-26T20:53:58.130151+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-26T20:52:13.796312+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"6b0f79e5-c540-4701-a03e-38b2596b333b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6802","nonce":2118194723},{"type":"v1","addr":"172.21.15.63:6803","nonce":2118194723}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6804","nonce":2118194723},{"type":"v1","addr":"172.21.15.63:6805","nonce":2118194723}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6808","nonce":2118194723},{"type":"v1","addr":"172.21.15.63:6809","nonce":2118194723}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6806","nonce":2118194723},{"type":"v1","addr":"172.21.15.63:6807","nonce":2118194723}]},"public_addr":"172.21.15.63:6803/2118194723","cluster_addr":"172.21.15.63:6805/2118194723","heartbeat_back_addr":"172.21.15.63:6809/2118194723","heartbeat_front_addr":"172.21.15.63:6807/2118194723","state":["exists","up"]},{"osd":1,"uuid":"cc56c3cf-5bdc-4d3b-b1fc-57f20f9c9509","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6810","nonce":4265583190},{"type":"v1","addr":"172.21.15.63:6811","nonce":4265583190}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6812","nonce":4265583190},{"type":"v1","addr":"172.21.15.63:6813","nonce":4265583190}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6816","nonce":4265583190},{"type":"v1","addr":"172.21.15.63:6817","nonce":4265583190}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6814","nonce":4265583190},{"type":"v1","addr":"172.21.15.63:6815","nonce":4265583190}]},"public_addr":"172.21.15.63:6811/4265583190","cluster_addr":"172.21.15.63:6813/4265583190","heartbeat_back_addr":"172.21.15.63:6817/4265583190","heartbeat_front_addr":"172.21.15.63:6815/4265583190","state":["exists","up"]},{"osd":2,"uuid":"5dcc41b7-668e-46c4-a389-e8a6cf2a5f64","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6818","nonce":2962020712},{"type":"v1","addr":"172.21.15.63:6819","nonce":2962020712}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6820","nonce":2962020712},{"type":"v1","addr":"172.21.15.63:6821","nonce":2962020712}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6824","nonce":2962020712},{"type":"v1","addr":"172.21.15.63:6825","nonce":2962020712}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6822","nonce":2962020712},{"type":"v1","addr":"172.21.15.63:6823","nonce":2962020712}]},"public_addr":"172.21.15.63:6819/2962020712","cluster_addr":"172.21.15.63:6821/2962020712","heartbeat_back_addr":"172.21.15.63:6825/2962020712","heartbeat_front_addr":"172.21.15.63:6823/2962020712","state":["exists","up"]},{"osd":3,"uuid":"21eb7156-f9a5-4108-982d-47847322b16e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6826","nonce":284512069},{"type":"v1","addr":"172.21.15.63:6827","nonce":284512069}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6828","nonce":284512069},{"type":"v1","addr":"172.21.15.63:6829","nonce":284512069}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6832","nonce":284512069},{"type":"v1","addr":"172.21.15.63:6833","nonce":284512069}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6830","nonce":284512069},{"type":"v1","addr":"172.21.15.63:6831","nonce":284512069}]},"public_addr":"172.21.15.63:6827/284512069","cluster_addr":"172.21.15.63:6829/284512069","heartbeat_back_addr":"172.21.15.63:6833/284512069","heartbeat_front_addr":"172.21.15.63:6831/284512069","state":["exists","up"]},{"osd":4,"uuid":"d2574505-8824-4246-95ac-83d1d8a0b357","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6800","nonce":2399757733},{"type":"v1","addr":"172.21.15.174:6801","nonce":2399757733}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6802","nonce":2399757733},{"type":"v1","addr":"172.21.15.174:6803","nonce":2399757733}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6806","nonce":2399757733},{"type":"v1","addr":"172.21.15.174:6807","nonce":2399757733}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6804","nonce":2399757733},{"type":"v1","addr":"172.21.15.174:6805","nonce":2399757733}]},"public_addr":"172.21.15.174:6801/2399757733","cluster_addr":"172.21.15.174:6803/2399757733","heartbeat_back_addr":"172.21.15.174:6807/2399757733","heartbeat_front_addr":"172.21.15.174:6805/2399757733","state":["exists","up"]},{"osd":5,"uuid":"b9bc8ccb-da0f-4bbc-88ef-ff861ee0892b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6808","nonce":1022923234},{"type":"v1","addr":"172.21.15.174:6809","nonce":1022923234}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6810","nonce":1022923234},{"type":"v1","addr":"172.21.15.174:6811","nonce":1022923234}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6814","nonce":1022923234},{"type":"v1","addr":"172.21.15.174:6815","nonce":1022923234}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6812","nonce":1022923234},{"type":"v1","addr":"172.21.15.174:6813","nonce":1022923234}]},"public_addr":"172.21.15.174:6809/1022923234","cluster_addr":"172.21.15.174:6811/1022923234","heartbeat_back_addr":"172.21.15.174:6815/1022923234","heartbeat_front_addr":"172.21.15.174:6813/1022923234","state":["exists","up"]},{"osd":6,"uuid":"2283b8d5-fd69-467a-ad0d-b3b546d1494d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6816","nonce":949715503},{"type":"v1","addr":"172.21.15.174:6817","nonce":949715503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6818","nonce":949715503},{"type":"v1","addr":"172.21.15.174:6819","nonce":949715503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6822","nonce":949715503},{"type":"v1","addr":"172.21.15.174:6823","nonce":949715503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6820","nonce":949715503},{"type":"v1","addr":"172.21.15.174:6821","nonce":949715503}]},"public_addr":"172.21.15.174:6817/949715503","cluster_addr":"172.21.15.174:6819/949715503","heartbeat_back_addr":"172.21.15.174:6823/949715503","heartbeat_front_addr":"172.21.15.174:6821/949715503","state":["exists","up"]},{"osd":7,"uuid":"adf5696d-40cb-4701-8766-738480428100","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6824","nonce":2501302609},{"type":"v1","addr":"172.21.15.174:6825","nonce":2501302609}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6826","nonce":2501302609},{"type":"v1","addr":"172.21.15.174:6827","nonce":2501302609}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6830","nonce":2501302609},{"type":"v1","addr":"172.21.15.174:6831","nonce":2501302609}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6828","nonce":2501302609},{"type":"v1","addr":"172.21.15.174:6829","nonce":2501302609}]},"public_addr":"172.21.15.174:6825/2501302609","cluster_addr":"172.21.15.174:6827/2501302609","heartbeat_back_addr":"172.21.15.174:6831/2501302609","heartbeat_front_addr":"172.21.15.174:6829/2501302609","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-26T20:51:11.316361+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-26T20:51:43.245122+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-26T20:52:11.736302+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-26T20:52:39.294718+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-26T20:53:01.346252+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-26T20:53:25.308475+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-26T20:53:49.290005+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-26T20:54:13.664236+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.63:0/2441187239":"2024-03-27T20:50:06.915397+0000","172.21.15.63:6800/2036968009":"2024-03-27T20:50:06.915397+0000","172.21.15.63:0/1102163987":"2024-03-27T20:50:06.915397+0000","172.21.15.63:0/3588541937":"2024-03-27T20:49:12.588686+0000","172.21.15.63:6801/2789315162":"2024-03-27T20:49:12.588686+0000","172.21.15.63:0/3031931285":"2024-03-27T20:50:06.915397+0000","172.21.15.63:6800/2789315162":"2024-03-27T20:49:12.588686+0000","172.21.15.63:0/2089226072":"2024-03-27T20:49:12.588686+0000","172.21.15.63:6801/2036968009":"2024-03-27T20:50:06.915397+0000","172.21.15.63:6800/3345547800":"2024-03-27T20:48:42.768729+0000","172.21.15.63:6801/3345547800":"2024-03-27T20:48:42.768729+0000","172.21.15.63:0/3861012011":"2024-03-27T20:48:42.768729+0000","172.21.15.63:0/798866617":"2024-03-27T20:49:12.588686+0000","172.21.15.63:0/1772414794":"2024-03-27T20:48:42.768729+0000","172.21.15.63:0/1448834781":"2024-03-27T20:48:42.768729+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-26T20:54:30.605 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-03-26T20:54:30.605 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -- ceph osd dump --format=json 2024-03-26T20:54:30.825 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/mon.smithi063/config 2024-03-26T20:54:30.946 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:30 smithi063 ceph-mon[24818]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-26T20:54:30.947 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:30 smithi063 ceph-mon[24818]: from='client.? 172.21.15.63:0/3673108809' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-26T20:54:31.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:30 smithi174 ceph-mon[26934]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-26T20:54:31.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:30 smithi174 ceph-mon[26934]: from='client.? 172.21.15.63:0/3673108809' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-26T20:54:32.647 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-03-26T20:54:32.648 INFO:teuthology.orchestra.run.smithi063.stdout:{"epoch":59,"fsid":"0ed4dd4c-ebb2-11ee-95d0-87774f69a715","created":"2024-03-26T20:48:14.448236+0000","modified":"2024-03-26T20:54:19.615732+0000","last_up_change":"2024-03-26T20:54:15.599230+0000","last_in_change":"2024-03-26T20:53:58.130151+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-26T20:52:13.796312+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"6b0f79e5-c540-4701-a03e-38b2596b333b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6802","nonce":2118194723},{"type":"v1","addr":"172.21.15.63:6803","nonce":2118194723}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6804","nonce":2118194723},{"type":"v1","addr":"172.21.15.63:6805","nonce":2118194723}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6808","nonce":2118194723},{"type":"v1","addr":"172.21.15.63:6809","nonce":2118194723}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6806","nonce":2118194723},{"type":"v1","addr":"172.21.15.63:6807","nonce":2118194723}]},"public_addr":"172.21.15.63:6803/2118194723","cluster_addr":"172.21.15.63:6805/2118194723","heartbeat_back_addr":"172.21.15.63:6809/2118194723","heartbeat_front_addr":"172.21.15.63:6807/2118194723","state":["exists","up"]},{"osd":1,"uuid":"cc56c3cf-5bdc-4d3b-b1fc-57f20f9c9509","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6810","nonce":4265583190},{"type":"v1","addr":"172.21.15.63:6811","nonce":4265583190}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6812","nonce":4265583190},{"type":"v1","addr":"172.21.15.63:6813","nonce":4265583190}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6816","nonce":4265583190},{"type":"v1","addr":"172.21.15.63:6817","nonce":4265583190}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6814","nonce":4265583190},{"type":"v1","addr":"172.21.15.63:6815","nonce":4265583190}]},"public_addr":"172.21.15.63:6811/4265583190","cluster_addr":"172.21.15.63:6813/4265583190","heartbeat_back_addr":"172.21.15.63:6817/4265583190","heartbeat_front_addr":"172.21.15.63:6815/4265583190","state":["exists","up"]},{"osd":2,"uuid":"5dcc41b7-668e-46c4-a389-e8a6cf2a5f64","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6818","nonce":2962020712},{"type":"v1","addr":"172.21.15.63:6819","nonce":2962020712}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6820","nonce":2962020712},{"type":"v1","addr":"172.21.15.63:6821","nonce":2962020712}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6824","nonce":2962020712},{"type":"v1","addr":"172.21.15.63:6825","nonce":2962020712}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6822","nonce":2962020712},{"type":"v1","addr":"172.21.15.63:6823","nonce":2962020712}]},"public_addr":"172.21.15.63:6819/2962020712","cluster_addr":"172.21.15.63:6821/2962020712","heartbeat_back_addr":"172.21.15.63:6825/2962020712","heartbeat_front_addr":"172.21.15.63:6823/2962020712","state":["exists","up"]},{"osd":3,"uuid":"21eb7156-f9a5-4108-982d-47847322b16e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6826","nonce":284512069},{"type":"v1","addr":"172.21.15.63:6827","nonce":284512069}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6828","nonce":284512069},{"type":"v1","addr":"172.21.15.63:6829","nonce":284512069}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6832","nonce":284512069},{"type":"v1","addr":"172.21.15.63:6833","nonce":284512069}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6830","nonce":284512069},{"type":"v1","addr":"172.21.15.63:6831","nonce":284512069}]},"public_addr":"172.21.15.63:6827/284512069","cluster_addr":"172.21.15.63:6829/284512069","heartbeat_back_addr":"172.21.15.63:6833/284512069","heartbeat_front_addr":"172.21.15.63:6831/284512069","state":["exists","up"]},{"osd":4,"uuid":"d2574505-8824-4246-95ac-83d1d8a0b357","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6800","nonce":2399757733},{"type":"v1","addr":"172.21.15.174:6801","nonce":2399757733}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6802","nonce":2399757733},{"type":"v1","addr":"172.21.15.174:6803","nonce":2399757733}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6806","nonce":2399757733},{"type":"v1","addr":"172.21.15.174:6807","nonce":2399757733}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6804","nonce":2399757733},{"type":"v1","addr":"172.21.15.174:6805","nonce":2399757733}]},"public_addr":"172.21.15.174:6801/2399757733","cluster_addr":"172.21.15.174:6803/2399757733","heartbeat_back_addr":"172.21.15.174:6807/2399757733","heartbeat_front_addr":"172.21.15.174:6805/2399757733","state":["exists","up"]},{"osd":5,"uuid":"b9bc8ccb-da0f-4bbc-88ef-ff861ee0892b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6808","nonce":1022923234},{"type":"v1","addr":"172.21.15.174:6809","nonce":1022923234}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6810","nonce":1022923234},{"type":"v1","addr":"172.21.15.174:6811","nonce":1022923234}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6814","nonce":1022923234},{"type":"v1","addr":"172.21.15.174:6815","nonce":1022923234}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6812","nonce":1022923234},{"type":"v1","addr":"172.21.15.174:6813","nonce":1022923234}]},"public_addr":"172.21.15.174:6809/1022923234","cluster_addr":"172.21.15.174:6811/1022923234","heartbeat_back_addr":"172.21.15.174:6815/1022923234","heartbeat_front_addr":"172.21.15.174:6813/1022923234","state":["exists","up"]},{"osd":6,"uuid":"2283b8d5-fd69-467a-ad0d-b3b546d1494d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6816","nonce":949715503},{"type":"v1","addr":"172.21.15.174:6817","nonce":949715503}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6818","nonce":949715503},{"type":"v1","addr":"172.21.15.174:6819","nonce":949715503}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6822","nonce":949715503},{"type":"v1","addr":"172.21.15.174:6823","nonce":949715503}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6820","nonce":949715503},{"type":"v1","addr":"172.21.15.174:6821","nonce":949715503}]},"public_addr":"172.21.15.174:6817/949715503","cluster_addr":"172.21.15.174:6819/949715503","heartbeat_back_addr":"172.21.15.174:6823/949715503","heartbeat_front_addr":"172.21.15.174:6821/949715503","state":["exists","up"]},{"osd":7,"uuid":"adf5696d-40cb-4701-8766-738480428100","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6824","nonce":2501302609},{"type":"v1","addr":"172.21.15.174:6825","nonce":2501302609}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6826","nonce":2501302609},{"type":"v1","addr":"172.21.15.174:6827","nonce":2501302609}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6830","nonce":2501302609},{"type":"v1","addr":"172.21.15.174:6831","nonce":2501302609}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6828","nonce":2501302609},{"type":"v1","addr":"172.21.15.174:6829","nonce":2501302609}]},"public_addr":"172.21.15.174:6825/2501302609","cluster_addr":"172.21.15.174:6827/2501302609","heartbeat_back_addr":"172.21.15.174:6831/2501302609","heartbeat_front_addr":"172.21.15.174:6829/2501302609","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-26T20:51:11.316361+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-26T20:51:43.245122+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-26T20:52:11.736302+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-26T20:52:39.294718+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-26T20:53:01.346252+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-26T20:53:25.308475+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-26T20:53:49.290005+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-03-26T20:54:13.664236+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.63:0/2441187239":"2024-03-27T20:50:06.915397+0000","172.21.15.63:6800/2036968009":"2024-03-27T20:50:06.915397+0000","172.21.15.63:0/1102163987":"2024-03-27T20:50:06.915397+0000","172.21.15.63:0/3588541937":"2024-03-27T20:49:12.588686+0000","172.21.15.63:6801/2789315162":"2024-03-27T20:49:12.588686+0000","172.21.15.63:0/3031931285":"2024-03-27T20:50:06.915397+0000","172.21.15.63:6800/2789315162":"2024-03-27T20:49:12.588686+0000","172.21.15.63:0/2089226072":"2024-03-27T20:49:12.588686+0000","172.21.15.63:6801/2036968009":"2024-03-27T20:50:06.915397+0000","172.21.15.63:6800/3345547800":"2024-03-27T20:48:42.768729+0000","172.21.15.63:6801/3345547800":"2024-03-27T20:48:42.768729+0000","172.21.15.63:0/3861012011":"2024-03-27T20:48:42.768729+0000","172.21.15.63:0/798866617":"2024-03-27T20:49:12.588686+0000","172.21.15.63:0/1772414794":"2024-03-27T20:48:42.768729+0000","172.21.15.63:0/1448834781":"2024-03-27T20:48:42.768729+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-26T20:54:32.950 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:32 smithi063 ceph-mon[24818]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-26T20:54:33.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:32 smithi174 ceph-mon[26934]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-26T20:54:33.269 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -- ceph tell osd.0 flush_pg_stats 2024-03-26T20:54:33.270 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -- ceph tell osd.1 flush_pg_stats 2024-03-26T20:54:33.270 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -- ceph tell osd.2 flush_pg_stats 2024-03-26T20:54:33.271 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -- ceph tell osd.3 flush_pg_stats 2024-03-26T20:54:33.271 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -- ceph tell osd.4 flush_pg_stats 2024-03-26T20:54:33.271 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -- ceph tell osd.5 flush_pg_stats 2024-03-26T20:54:33.272 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -- ceph tell osd.6 flush_pg_stats 2024-03-26T20:54:33.272 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -- ceph tell osd.7 flush_pg_stats 2024-03-26T20:54:33.724 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/mon.smithi063/config 2024-03-26T20:54:33.725 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/mon.smithi063/config 2024-03-26T20:54:33.727 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/mon.smithi063/config 2024-03-26T20:54:33.727 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/mon.smithi063/config 2024-03-26T20:54:33.727 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/mon.smithi063/config 2024-03-26T20:54:33.728 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/mon.smithi063/config 2024-03-26T20:54:33.728 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/mon.smithi063/config 2024-03-26T20:54:33.732 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/mon.smithi063/config 2024-03-26T20:54:33.937 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:33 smithi063 ceph-mon[24818]: from='client.? 172.21.15.63:0/2147887512' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-26T20:54:34.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:33 smithi174 ceph-mon[26934]: from='client.? 172.21.15.63:0/2147887512' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-26T20:54:34.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:34 smithi063 ceph-mon[24818]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-26T20:54:34.902 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:34 smithi174 ceph-mon[26934]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-26T20:54:36.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:36 smithi063 ceph-mon[24818]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-26T20:54:37.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:36 smithi174 ceph-mon[26934]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-26T20:54:38.551 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:38 smithi063 ceph-mon[24818]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-26T20:54:38.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:38 smithi174 ceph-mon[26934]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-26T20:54:41.027 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:40 smithi063 ceph-mon[24818]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-26T20:54:41.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:40 smithi174 ceph-mon[26934]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-26T20:54:43.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:42 smithi174 ceph-mon[26934]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-26T20:54:43.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:42 smithi063 ceph-mon[24818]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-26T20:54:44.056 INFO:teuthology.orchestra.run.smithi063.stdout:111669149724 2024-03-26T20:54:44.056 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -- ceph osd last-stat-seq osd.3 2024-03-26T20:54:44.670 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:44 smithi063 ceph-mon[24818]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-26T20:54:45.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:44 smithi174 ceph-mon[26934]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-26T20:54:46.598 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:46 smithi063 ceph-mon[24818]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-26T20:54:46.623 INFO:teuthology.orchestra.run.smithi063.stdout:81604378657 2024-03-26T20:54:46.623 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -- ceph osd last-stat-seq osd.2 2024-03-26T20:54:47.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:46 smithi174 ceph-mon[26934]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-26T20:54:47.996 INFO:teuthology.orchestra.run.smithi063.stdout:236223201289 2024-03-26T20:54:47.996 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -- ceph osd last-stat-seq osd.7 2024-03-26T20:54:48.551 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:48 smithi063 ceph-mon[24818]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-26T20:54:48.683 INFO:teuthology.orchestra.run.smithi063.stdout:60129542183 2024-03-26T20:54:48.683 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -- ceph osd last-stat-seq osd.1 2024-03-26T20:54:48.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:48 smithi174 ceph-mon[26934]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-26T20:54:49.103 INFO:teuthology.orchestra.run.smithi063.stdout:133143986199 2024-03-26T20:54:49.103 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -- ceph osd last-stat-seq osd.4 2024-03-26T20:54:49.457 INFO:teuthology.orchestra.run.smithi063.stdout:167503724563 2024-03-26T20:54:49.457 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -- ceph osd last-stat-seq osd.5 2024-03-26T20:54:50.137 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/mon.smithi063/config 2024-03-26T20:54:50.666 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/mon.smithi063/config 2024-03-26T20:54:50.667 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/mon.smithi063/config 2024-03-26T20:54:50.667 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/mon.smithi063/config 2024-03-26T20:54:50.733 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:50 smithi063 ceph-mon[24818]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-26T20:54:50.735 INFO:teuthology.orchestra.run.smithi063.stdout:38654705711 2024-03-26T20:54:50.735 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -- ceph osd last-stat-seq osd.0 2024-03-26T20:54:51.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:50 smithi174 ceph-mon[26934]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-26T20:54:51.328 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/mon.smithi063/config 2024-03-26T20:54:51.349 INFO:teuthology.orchestra.run.smithi063.stdout:201863462926 2024-03-26T20:54:51.350 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -- ceph osd last-stat-seq osd.6 2024-03-26T20:54:52.985 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:52 smithi063 ceph-mon[24818]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-26T20:54:53.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:52 smithi174 ceph-mon[26934]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-26T20:54:54.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:54 smithi063 ceph-mon[24818]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-26T20:54:54.850 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:54 smithi174 ceph-mon[26934]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-26T20:54:55.550 INFO:teuthology.orchestra.run.smithi063.stdout:111669149726 2024-03-26T20:54:55.791 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:55 smithi063 ceph-mon[24818]: from='client.? 172.21.15.63:0/2260100331' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-03-26T20:54:56.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:55 smithi174 ceph-mon[26934]: from='client.? 172.21.15.63:0/2260100331' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-03-26T20:54:56.633 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/mon.smithi063/config 2024-03-26T20:54:56.634 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/mon.smithi063/config 2024-03-26T20:54:56.635 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/mon.smithi063/config 2024-03-26T20:54:56.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:56 smithi063 ceph-mon[24818]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-26T20:54:56.832 INFO:teuthology.orchestra.run.smithi063.stdout:60129542185 2024-03-26T20:54:57.050 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:56 smithi174 ceph-mon[26934]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-26T20:54:57.695 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:57 smithi063 ceph-mon[24818]: from='client.? 172.21.15.63:0/3515696879' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-03-26T20:54:58.009 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149724 got 111669149726 for osd.3 2024-03-26T20:54:58.010 DEBUG:teuthology.parallel:result is None 2024-03-26T20:54:58.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:57 smithi174 ceph-mon[26934]: from='client.? 172.21.15.63:0/3515696879' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-03-26T20:54:58.747 INFO:teuthology.orchestra.run.smithi063.stdout:133143986201 2024-03-26T20:54:58.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:58 smithi063 ceph-mon[24818]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-26T20:54:59.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:58 smithi174 ceph-mon[26934]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-26T20:54:59.287 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986199 got 133143986201 for osd.4 2024-03-26T20:54:59.287 DEBUG:teuthology.parallel:result is None 2024-03-26T20:54:59.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:54:59 smithi063 ceph-mon[24818]: from='client.? 172.21.15.63:0/2435350138' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-03-26T20:55:00.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:54:59 smithi174 ceph-mon[26934]: from='client.? 172.21.15.63:0/2435350138' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-03-26T20:55:00.663 INFO:teuthology.orchestra.run.smithi063.stdout:236223201292 2024-03-26T20:55:00.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:00 smithi063 ceph-mon[24818]: pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-26T20:55:00.837 INFO:teuthology.orchestra.run.smithi063.stdout:167503724566 2024-03-26T20:55:01.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:00 smithi174 ceph-mon[26934]: pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-26T20:55:01.643 INFO:tasks.cephadm.ceph_manager.ceph:need seq 60129542183 got 60129542185 for osd.1 2024-03-26T20:55:01.644 DEBUG:teuthology.parallel:result is None 2024-03-26T20:55:01.946 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:01 smithi063 ceph-mon[24818]: from='client.? 172.21.15.63:0/528950700' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-03-26T20:55:01.946 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:01 smithi063 ceph-mon[24818]: from='client.? 172.21.15.63:0/356282939' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-03-26T20:55:01.972 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201289 got 236223201292 for osd.7 2024-03-26T20:55:01.972 DEBUG:teuthology.parallel:result is None 2024-03-26T20:55:02.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:01 smithi174 ceph-mon[26934]: from='client.? 172.21.15.63:0/528950700' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-03-26T20:55:02.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:01 smithi174 ceph-mon[26934]: from='client.? 172.21.15.63:0/356282939' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-03-26T20:55:03.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:02 smithi174 ceph-mon[26934]: pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-26T20:55:03.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:02 smithi063 ceph-mon[24818]: pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-26T20:55:03.448 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724563 got 167503724566 for osd.5 2024-03-26T20:55:03.448 DEBUG:teuthology.parallel:result is None 2024-03-26T20:55:04.360 INFO:teuthology.orchestra.run.smithi063.stdout:81604378660 2024-03-26T20:55:04.839 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:04 smithi063 ceph-mon[24818]: pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-26T20:55:04.839 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:04 smithi063 ceph-mon[24818]: from='client.? 172.21.15.63:0/1433469783' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-03-26T20:55:05.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:04 smithi174 ceph-mon[26934]: pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-26T20:55:05.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:04 smithi174 ceph-mon[26934]: from='client.? 172.21.15.63:0/1433469783' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-03-26T20:55:05.208 INFO:teuthology.orchestra.run.smithi063.stdout:201863462929 2024-03-26T20:55:06.028 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:05 smithi063 ceph-mon[24818]: from='client.? 172.21.15.63:0/1539231265' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-03-26T20:55:06.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:05 smithi174 ceph-mon[26934]: from='client.? 172.21.15.63:0/1539231265' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-03-26T20:55:06.051 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863462926 got 201863462929 for osd.6 2024-03-26T20:55:06.051 DEBUG:teuthology.parallel:result is None 2024-03-26T20:55:06.657 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:06 smithi063 ceph-mon[24818]: pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-26T20:55:06.683 INFO:tasks.cephadm.ceph_manager.ceph:need seq 81604378657 got 81604378660 for osd.2 2024-03-26T20:55:06.683 DEBUG:teuthology.parallel:result is None 2024-03-26T20:55:07.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:06 smithi174 ceph-mon[26934]: pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-26T20:55:07.290 INFO:teuthology.orchestra.run.smithi063.stdout:38654705714 2024-03-26T20:55:07.929 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:07 smithi063 ceph-mon[24818]: from='client.? 172.21.15.63:0/783724558' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-03-26T20:55:07.929 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:07 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi063.lxhujt/mirror_snapshot_schedule"}]: dispatch 2024-03-26T20:55:07.929 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:07 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi063.lxhujt/trash_purge_schedule"}]: dispatch 2024-03-26T20:55:07.952 INFO:tasks.cephadm.ceph_manager.ceph:need seq 38654705711 got 38654705714 for osd.0 2024-03-26T20:55:07.953 DEBUG:teuthology.parallel:result is None 2024-03-26T20:55:07.953 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-03-26T20:55:07.953 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -- ceph pg dump --format=json 2024-03-26T20:55:08.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:07 smithi174 ceph-mon[26934]: from='client.? 172.21.15.63:0/783724558' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-03-26T20:55:08.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:07 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi063.lxhujt/mirror_snapshot_schedule"}]: dispatch 2024-03-26T20:55:08.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:07 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi063.lxhujt/trash_purge_schedule"}]: dispatch 2024-03-26T20:55:08.172 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/mon.smithi063/config 2024-03-26T20:55:08.911 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:08 smithi063 ceph-mon[24818]: pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-26T20:55:09.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:08 smithi174 ceph-mon[26934]: pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-26T20:55:09.855 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-03-26T20:55:09.856 INFO:teuthology.orchestra.run.smithi063.stderr:dumped all 2024-03-26T20:55:10.499 INFO:teuthology.orchestra.run.smithi063.stdout:{"pg_ready":true,"pg_map":{"version":198,"stamp":"2024-03-26T20:55:09.512914+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48868,"kb_used_data":6308,"kb_used_omap":0,"kb_used_meta":42496,"kb_avail":749912348,"statfs":{"total":767960285184,"available":767910244352,"internally_reserved":0,"allocated":6459392,"data_stored":3547265,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43515904},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002521"},"pg_stats":[{"pgid":"1.0","version":"22'117","reported_seq":226,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-03-26T20:54:19.627819+0000","last_change":"2024-03-26T20:54:19.627819+0000","last_active":"2024-03-26T20:54:19.627819+0000","last_peered":"2024-03-26T20:54:19.627819+0000","last_clean":"2024-03-26T20:54:19.627819+0000","last_became_active":"2024-03-26T20:54:19.627351+0000","last_became_peered":"2024-03-26T20:54:19.627351+0000","last_unstale":"2024-03-26T20:54:19.627819+0000","last_undegraded":"2024-03-26T20:54:19.627819+0000","last_fullsized":"2024-03-26T20:54:19.627819+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-26T20:52:14.542527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-26T20:52:14.542527+0000","last_clean_scrub_stamp":"2024-03-26T20:52:14.542527+0000","objects_scrubbed":0,"log_size":117,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-28T01:01:22.335788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3727360,"data_stored":3702496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201294,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6052,"kb_used_data":1116,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739100,"statfs":{"total":95995035648,"available":95988838400,"internally_reserved":0,"allocated":1142784,"data_stored":774161,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.66000000000000003}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.72699999999999998}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.68500000000000005}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.69799999999999995}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57599999999999996}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.59699999999999998}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.70699999999999996}]}]},{"osd":6,"up_from":47,"seq":201863462930,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6176,"kb_used_data":1112,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93738976,"statfs":{"total":95995035648,"available":95988711424,"internally_reserved":0,"allocated":1138688,"data_stored":773896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Mar 26 20:54:54 2024","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.55000000000000004}]},{"osd":1,"last update":"Tue Mar 26 20:54:54 2024","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.81599999999999995,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.54400000000000004}]},{"osd":2,"last update":"Tue Mar 26 20:54:54 2024","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.46500000000000002}]},{"osd":3,"last update":"Tue Mar 26 20:54:54 2024","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.53000000000000003},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.44800000000000001}]},{"osd":4,"last update":"Tue Mar 26 20:54:54 2024","interfaces":[{"interface":"back","average":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.746,"5min":0.746,"15min":0.746},"last":0.58099999999999996},{"interface":"front","average":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.47999999999999998}]},{"osd":5,"last update":"Tue Mar 26 20:54:54 2024","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.749,"5min":0.749,"15min":0.749},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.38200000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.60099999999999998}]}]},{"osd":1,"up_from":14,"seq":60129542188,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6360,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738792,"statfs":{"total":95995035648,"available":95988523008,"internally_reserved":0,"allocated":606208,"data_stored":244968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Mar 26 20:54:53 2024","interfaces":[{"interface":"back","average":{"1min":0.36599999999999999,"5min":0.33600000000000002,"15min":0.314},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.54600000000000004,"5min":0.58499999999999996,"15min":0.58499999999999996},"last":0.38400000000000001},{"interface":"front","average":{"1min":0.39300000000000002,"5min":0.35299999999999998,"15min":0.32900000000000001},"min":{"1min":0.24099999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.51500000000000001,"5min":0.54100000000000004,"15min":0.54100000000000004},"last":0.41299999999999998}]},{"osd":2,"last update":"Tue Mar 26 20:54:18 2024","interfaces":[{"interface":"back","average":{"1min":0.45000000000000001,"5min":0.39600000000000002,"15min":0.38700000000000001},"min":{"1min":0.33200000000000002,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.33000000000000002},{"interface":"front","average":{"1min":0.44600000000000001,"5min":0.39400000000000002,"15min":0.38500000000000001},"min":{"1min":0.33700000000000002,"5min":0.25,"15min":0.25},"max":{"1min":0.54000000000000004,"5min":0.59899999999999998,"15min":0.59899999999999998},"last":0.44600000000000001}]},{"osd":3,"last update":"Tue Mar 26 20:54:43 2024","interfaces":[{"interface":"back","average":{"1min":0.39400000000000002,"5min":0.436,"15min":0.443},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.56599999999999995,"5min":0.64500000000000002,"15min":0.64500000000000002},"last":0.36199999999999999},{"interface":"front","average":{"1min":0.42299999999999999,"5min":0.443,"15min":0.44700000000000001},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.53400000000000003,"5min":0.61499999999999999,"15min":0.61499999999999999},"last":0.42499999999999999}]},{"osd":4,"last update":"Tue Mar 26 20:54:05 2024","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.68899999999999995,"5min":0.68899999999999995,"15min":0.68899999999999995},"last":0.625},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.90000000000000002}]},{"osd":5,"last update":"Tue Mar 26 20:54:27 2024","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"min":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.99099999999999999},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"min":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"max":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.65800000000000003}]},{"osd":6,"last update":"Tue Mar 26 20:54:53 2024","interfaces":[{"interface":"back","average":{"1min":1.1910000000000001,"5min":1.1910000000000001,"15min":1.1910000000000001},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":13.438000000000001,"5min":13.438000000000001,"15min":13.438000000000001},"last":0.84199999999999997},{"interface":"front","average":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":1.0900000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.80700000000000005}]}]},{"osd":0,"up_from":9,"seq":38654705715,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6884,"kb_used_data":1116,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738268,"statfs":{"total":95995035648,"available":95987986432,"internally_reserved":0,"allocated":1142784,"data_stored":774161,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Tue Mar 26 20:54:53 2024","interfaces":[{"interface":"back","average":{"1min":0.39800000000000002,"5min":0.34000000000000002,"15min":0.314},"min":{"1min":0.27100000000000002,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.374,"5min":0.33600000000000002,"15min":0.312},"min":{"1min":0.23799999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"last":0.434}]},{"osd":2,"last update":"Tue Mar 26 20:54:19 2024","interfaces":[{"interface":"back","average":{"1min":0.40600000000000003,"5min":0.39500000000000002,"15min":0.39300000000000002},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.55200000000000005,"5min":0.61599999999999999,"15min":0.61599999999999999},"last":0.35899999999999999},{"interface":"front","average":{"1min":0.43099999999999999,"5min":0.40300000000000002,"15min":0.39800000000000002},"min":{"1min":0.27600000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"last":0.48599999999999999}]},{"osd":3,"last update":"Tue Mar 26 20:54:44 2024","interfaces":[{"interface":"back","average":{"1min":0.41299999999999998,"5min":0.46100000000000002,"15min":0.46899999999999997},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.53900000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.433,"5min":0.46500000000000002,"15min":0.47099999999999997},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.60699999999999998,"5min":0.622,"15min":0.622},"last":0.50600000000000001}]},{"osd":4,"last update":"Tue Mar 26 20:55:07 2024","interfaces":[{"interface":"back","average":{"1min":0.64700000000000002,"5min":0.59899999999999998,"15min":0.59099999999999997},"min":{"1min":0.45100000000000001,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.89200000000000002,"5min":0.89200000000000002,"15min":0.89200000000000002},"last":0.76700000000000002},{"interface":"front","average":{"1min":0.67100000000000004,"5min":0.61099999999999999,"15min":0.60099999999999998},"min":{"1min":0.50600000000000001,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.86599999999999999,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.68200000000000005}]},{"osd":5,"last update":"Tue Mar 26 20:54:28 2024","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":1.127,"5min":1.127,"15min":1.127},"last":0.78100000000000003},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.70499999999999996}]},{"osd":6,"last update":"Tue Mar 26 20:54:53 2024","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":0.84899999999999998,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.66100000000000003},{"interface":"front","average":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"min":{"1min":0.499,"5min":0.499,"15min":0.499},"max":{"1min":0.83499999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.746}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.80600000000000005}]}]},{"osd":2,"up_from":19,"seq":81604378661,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6236,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5632,"kb_avail":93738916,"statfs":{"total":95995035648,"available":95988649984,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5767168},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Mar 26 20:54:18 2024","interfaces":[{"interface":"back","average":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"last":0.29399999999999998},{"interface":"front","average":{"1min":0.47299999999999998,"5min":0.40699999999999997,"15min":0.39600000000000002},"min":{"1min":0.308,"5min":0.25,"15min":0.25},"max":{"1min":0.96699999999999997,"5min":0.96699999999999997,"15min":0.96699999999999997},"last":0.44}]},{"osd":1,"last update":"Tue Mar 26 20:54:18 2024","interfaces":[{"interface":"back","average":{"1min":0.42699999999999999,"5min":0.42199999999999999,"15min":0.42099999999999999},"min":{"1min":0.311,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"last":0.33800000000000002},{"interface":"front","average":{"1min":0.42699999999999999,"5min":0.39500000000000002,"15min":0.39000000000000001},"min":{"1min":0.30299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":0.67500000000000004,"5min":0.67500000000000004,"15min":0.67500000000000004},"last":0.34399999999999997}]},{"osd":3,"last update":"Tue Mar 26 20:54:46 2024","interfaces":[{"interface":"back","average":{"1min":0.41699999999999998,"5min":0.44500000000000001,"15min":0.44900000000000001},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":0.318},{"interface":"front","average":{"1min":0.47699999999999998,"5min":0.48399999999999999,"15min":0.48499999999999999},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.378}]},{"osd":4,"last update":"Tue Mar 26 20:54:07 2024","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.67700000000000005},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.89800000000000002,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":0.72699999999999998}]},{"osd":5,"last update":"Tue Mar 26 20:54:27 2024","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.64900000000000002},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.64200000000000002}]},{"osd":6,"last update":"Tue Mar 26 20:54:54 2024","interfaces":[{"interface":"back","average":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"min":{"1min":0.5,"5min":0.5,"15min":0.5},"max":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.54300000000000004},{"interface":"front","average":{"1min":0.66600000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"min":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"max":{"1min":0.94599999999999995,"5min":0.94599999999999995,"15min":0.94599999999999995},"last":0.60299999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58899999999999997}]}]},{"osd":3,"up_from":26,"seq":111669149729,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5720,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739432,"statfs":{"total":95995035648,"available":95989178368,"internally_reserved":0,"allocated":606208,"data_stored":244968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Mar 26 20:54:43 2024","interfaces":[{"interface":"back","average":{"1min":0.37,"5min":0.39800000000000002,"15min":0.40300000000000002},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.56499999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"last":0.247},{"interface":"front","average":{"1min":0.38500000000000001,"5min":0.41799999999999998,"15min":0.42399999999999999},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":0.47999999999999998,"5min":0.55100000000000005,"15min":0.55100000000000005},"last":0.34599999999999997}]},{"osd":1,"last update":"Tue Mar 26 20:54:43 2024","interfaces":[{"interface":"back","average":{"1min":0.42099999999999999,"5min":0.44,"15min":0.443},"min":{"1min":0.32000000000000001,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.58399999999999996,"5min":0.61399999999999999,"15min":0.61399999999999999},"last":0.29499999999999998},{"interface":"front","average":{"1min":0.39100000000000001,"5min":0.43099999999999999,"15min":0.438},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.54200000000000004,"5min":0.60099999999999998,"15min":0.60099999999999998},"last":0.44400000000000001}]},{"osd":2,"last update":"Tue Mar 26 20:54:43 2024","interfaces":[{"interface":"back","average":{"1min":0.47499999999999998,"5min":0.48499999999999999,"15min":0.48699999999999999},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.58399999999999996,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.33000000000000002},{"interface":"front","average":{"1min":0.45600000000000002,"5min":0.47499999999999998,"15min":0.47799999999999998},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.58999999999999997,"5min":0.65300000000000002,"15min":0.65300000000000002},"last":0.27800000000000002}]},{"osd":4,"last update":"Tue Mar 26 20:54:05 2024","interfaces":[{"interface":"back","average":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.621,"5min":0.621,"15min":0.621},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.53800000000000003}]},{"osd":5,"last update":"Tue Mar 26 20:54:31 2024","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":1.1990000000000001,"5min":1.1990000000000001,"15min":1.1990000000000001},"last":0.51000000000000001},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.55100000000000005}]},{"osd":6,"last update":"Tue Mar 26 20:54:53 2024","interfaces":[{"interface":"back","average":{"1min":1.0209999999999999,"5min":1.0209999999999999,"15min":1.0209999999999999},"min":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":6.4699999999999998,"5min":6.4699999999999998,"15min":6.4699999999999998},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"min":{"1min":0.504,"5min":0.504,"15min":0.504},"max":{"1min":0.81399999999999995,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.56899999999999995}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51900000000000002}]}]},{"osd":4,"up_from":31,"seq":133143986204,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5784,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739368,"statfs":{"total":95995035648,"available":95989112832,"internally_reserved":0,"allocated":606208,"data_stored":244910,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Mar 26 20:54:04 2024","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.48699999999999999,"15min":0.48699999999999999},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.73999999999999999},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":2.6819999999999999,"5min":2.6819999999999999,"15min":2.6819999999999999},"last":0.61199999999999999}]},{"osd":1,"last update":"Tue Mar 26 20:54:04 2024","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.62},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.68400000000000005}]},{"osd":2,"last update":"Tue Mar 26 20:54:04 2024","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.629,"5min":0.629,"15min":0.629},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":2.7280000000000002,"5min":2.7280000000000002,"15min":2.7280000000000002},"last":0.66200000000000003}]},{"osd":3,"last update":"Tue Mar 26 20:54:04 2024","interfaces":[{"interface":"back","average":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":2.7109999999999999,"5min":2.7109999999999999,"15min":2.7109999999999999},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":2.653,"5min":2.653,"15min":2.653},"last":0.67300000000000004}]},{"osd":5,"last update":"Tue Mar 26 20:54:29 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":2.6379999999999999,"5min":2.6379999999999999,"15min":2.6379999999999999},"last":0.751},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.97199999999999998,"5min":0.97199999999999998,"15min":0.97199999999999998},"last":0.70799999999999996}]},{"osd":6,"last update":"Tue Mar 26 20:54:55 2024","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":1.0109999999999999,"5min":1.0109999999999999,"15min":1.0109999999999999},"last":0.65000000000000002},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.98699999999999999,"5min":0.98699999999999999,"15min":0.98699999999999999},"last":0.79500000000000004}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.76500000000000001}]}]},{"osd":5,"up_from":39,"seq":167503724568,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5656,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739496,"statfs":{"total":95995035648,"available":95989243904,"internally_reserved":0,"allocated":606208,"data_stored":244968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Mar 26 20:54:29 2024","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.501,"15min":0.501},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.61299999999999999},{"interface":"front","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.59399999999999997}]},{"osd":1,"last update":"Tue Mar 26 20:54:29 2024","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.73299999999999998},{"interface":"front","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.64900000000000002}]},{"osd":2,"last update":"Tue Mar 26 20:54:29 2024","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.65800000000000003},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.69399999999999995}]},{"osd":3,"last update":"Tue Mar 26 20:54:29 2024","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.749},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.83099999999999996,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.63600000000000001}]},{"osd":4,"last update":"Tue Mar 26 20:54:29 2024","interfaces":[{"interface":"back","average":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.629,"5min":0.629,"15min":0.629},"last":0.625},{"interface":"front","average":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.68400000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.57599999999999996}]},{"osd":6,"last update":"Tue Mar 26 20:54:51 2024","interfaces":[{"interface":"back","average":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.54600000000000004}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.60299999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-26T20:55:10.501 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -- ceph pg dump --format=json 2024-03-26T20:55:10.720 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/mon.smithi063/config 2024-03-26T20:55:10.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:10 smithi063 ceph-mon[24818]: pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-26T20:55:10.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:10 smithi063 ceph-mon[24818]: from='client.14438 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-26T20:55:11.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:10 smithi174 ceph-mon[26934]: pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-26T20:55:11.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:10 smithi174 ceph-mon[26934]: from='client.14438 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-26T20:55:12.451 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-03-26T20:55:12.452 INFO:teuthology.orchestra.run.smithi063.stderr:dumped all 2024-03-26T20:55:12.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:12 smithi063 ceph-mon[24818]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-26T20:55:13.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:12 smithi174 ceph-mon[26934]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-26T20:55:13.085 INFO:teuthology.orchestra.run.smithi063.stdout:{"pg_ready":true,"pg_map":{"version":199,"stamp":"2024-03-26T20:55:11.513452+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48868,"kb_used_data":6308,"kb_used_omap":0,"kb_used_meta":42496,"kb_avail":749912348,"statfs":{"total":767960285184,"available":767910244352,"internally_reserved":0,"allocated":6459392,"data_stored":3547265,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43515904},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002697"},"pg_stats":[{"pgid":"1.0","version":"22'117","reported_seq":226,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-03-26T20:54:19.627819+0000","last_change":"2024-03-26T20:54:19.627819+0000","last_active":"2024-03-26T20:54:19.627819+0000","last_peered":"2024-03-26T20:54:19.627819+0000","last_clean":"2024-03-26T20:54:19.627819+0000","last_became_active":"2024-03-26T20:54:19.627351+0000","last_became_peered":"2024-03-26T20:54:19.627351+0000","last_unstale":"2024-03-26T20:54:19.627819+0000","last_undegraded":"2024-03-26T20:54:19.627819+0000","last_fullsized":"2024-03-26T20:54:19.627819+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-26T20:52:14.542527+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-26T20:52:14.542527+0000","last_clean_scrub_stamp":"2024-03-26T20:52:14.542527+0000","objects_scrubbed":0,"log_size":117,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-28T01:01:22.335788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3727360,"data_stored":3702496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201294,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6052,"kb_used_data":1116,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739100,"statfs":{"total":95995035648,"available":95988838400,"internally_reserved":0,"allocated":1142784,"data_stored":774161,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.66000000000000003}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.72699999999999998}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.68500000000000005}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.69799999999999995}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57599999999999996}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.59699999999999998}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.70699999999999996}]}]},{"osd":6,"up_from":47,"seq":201863462931,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6176,"kb_used_data":1112,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93738976,"statfs":{"total":95995035648,"available":95988711424,"internally_reserved":0,"allocated":1138688,"data_stored":773896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Mar 26 20:54:54 2024","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.33400000000000002},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.374}]},{"osd":1,"last update":"Tue Mar 26 20:54:54 2024","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.81599999999999995,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.40000000000000002}]},{"osd":2,"last update":"Tue Mar 26 20:54:54 2024","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.49299999999999999},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.435}]},{"osd":3,"last update":"Tue Mar 26 20:54:54 2024","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.63},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.45500000000000002}]},{"osd":4,"last update":"Tue Mar 26 20:54:54 2024","interfaces":[{"interface":"back","average":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.746,"5min":0.746,"15min":0.746},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.60499999999999998}]},{"osd":5,"last update":"Tue Mar 26 20:54:54 2024","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.749,"5min":0.749,"15min":0.749},"last":0.52100000000000002},{"interface":"front","average":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.65800000000000003}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.48299999999999998}]}]},{"osd":1,"up_from":14,"seq":60129542189,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6360,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738792,"statfs":{"total":95995035648,"available":95988523008,"internally_reserved":0,"allocated":606208,"data_stored":244968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Mar 26 20:54:53 2024","interfaces":[{"interface":"back","average":{"1min":0.36599999999999999,"5min":0.33600000000000002,"15min":0.314},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.54600000000000004,"5min":0.58499999999999996,"15min":0.58499999999999996},"last":0.374},{"interface":"front","average":{"1min":0.39300000000000002,"5min":0.35299999999999998,"15min":0.32900000000000001},"min":{"1min":0.24099999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.51500000000000001,"5min":0.54100000000000004,"15min":0.54100000000000004},"last":0.40200000000000002}]},{"osd":2,"last update":"Tue Mar 26 20:54:18 2024","interfaces":[{"interface":"back","average":{"1min":0.45000000000000001,"5min":0.39600000000000002,"15min":0.38700000000000001},"min":{"1min":0.33200000000000002,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.36699999999999999},{"interface":"front","average":{"1min":0.44600000000000001,"5min":0.39400000000000002,"15min":0.38500000000000001},"min":{"1min":0.33700000000000002,"5min":0.25,"15min":0.25},"max":{"1min":0.54000000000000004,"5min":0.59899999999999998,"15min":0.59899999999999998},"last":0.29999999999999999}]},{"osd":3,"last update":"Tue Mar 26 20:54:43 2024","interfaces":[{"interface":"back","average":{"1min":0.39400000000000002,"5min":0.436,"15min":0.443},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.56599999999999995,"5min":0.64500000000000002,"15min":0.64500000000000002},"last":0.34799999999999998},{"interface":"front","average":{"1min":0.42299999999999999,"5min":0.443,"15min":0.44700000000000001},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.53400000000000003,"5min":0.61499999999999999,"15min":0.61499999999999999},"last":0.33300000000000002}]},{"osd":4,"last update":"Tue Mar 26 20:55:08 2024","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.58199999999999996,"15min":0.57899999999999996},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.89700000000000002,"5min":0.89700000000000002,"15min":0.89700000000000002},"last":0.54600000000000004},{"interface":"front","average":{"1min":1.1890000000000001,"5min":0.70399999999999996,"15min":0.623},"min":{"1min":0.48299999999999998,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":13.417,"5min":13.417,"15min":13.417},"last":0.58899999999999997}]},{"osd":5,"last update":"Tue Mar 26 20:54:27 2024","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"min":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.66200000000000003},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"min":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"max":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.59799999999999998}]},{"osd":6,"last update":"Tue Mar 26 20:54:53 2024","interfaces":[{"interface":"back","average":{"1min":1.1910000000000001,"5min":1.1910000000000001,"15min":1.1910000000000001},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":13.438000000000001,"5min":13.438000000000001,"15min":13.438000000000001},"last":0.61599999999999999},{"interface":"front","average":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.68200000000000005}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.61299999999999999}]}]},{"osd":0,"up_from":9,"seq":38654705715,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6884,"kb_used_data":1116,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738268,"statfs":{"total":95995035648,"available":95987986432,"internally_reserved":0,"allocated":1142784,"data_stored":774161,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Tue Mar 26 20:54:53 2024","interfaces":[{"interface":"back","average":{"1min":0.39800000000000002,"5min":0.34000000000000002,"15min":0.314},"min":{"1min":0.27100000000000002,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.374,"5min":0.33600000000000002,"15min":0.312},"min":{"1min":0.23799999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"last":0.434}]},{"osd":2,"last update":"Tue Mar 26 20:54:19 2024","interfaces":[{"interface":"back","average":{"1min":0.40600000000000003,"5min":0.39500000000000002,"15min":0.39300000000000002},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.55200000000000005,"5min":0.61599999999999999,"15min":0.61599999999999999},"last":0.35899999999999999},{"interface":"front","average":{"1min":0.43099999999999999,"5min":0.40300000000000002,"15min":0.39800000000000002},"min":{"1min":0.27600000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"last":0.48599999999999999}]},{"osd":3,"last update":"Tue Mar 26 20:54:44 2024","interfaces":[{"interface":"back","average":{"1min":0.41299999999999998,"5min":0.46100000000000002,"15min":0.46899999999999997},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.53900000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.433,"5min":0.46500000000000002,"15min":0.47099999999999997},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.60699999999999998,"5min":0.622,"15min":0.622},"last":0.50600000000000001}]},{"osd":4,"last update":"Tue Mar 26 20:55:07 2024","interfaces":[{"interface":"back","average":{"1min":0.64700000000000002,"5min":0.59899999999999998,"15min":0.59099999999999997},"min":{"1min":0.45100000000000001,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.89200000000000002,"5min":0.89200000000000002,"15min":0.89200000000000002},"last":0.76700000000000002},{"interface":"front","average":{"1min":0.67100000000000004,"5min":0.61099999999999999,"15min":0.60099999999999998},"min":{"1min":0.50600000000000001,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.86599999999999999,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.68200000000000005}]},{"osd":5,"last update":"Tue Mar 26 20:54:28 2024","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":1.127,"5min":1.127,"15min":1.127},"last":0.78100000000000003},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.70499999999999996}]},{"osd":6,"last update":"Tue Mar 26 20:54:53 2024","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":0.84899999999999998,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.66100000000000003},{"interface":"front","average":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"min":{"1min":0.499,"5min":0.499,"15min":0.499},"max":{"1min":0.83499999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.746}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.80600000000000005}]}]},{"osd":2,"up_from":19,"seq":81604378662,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6236,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5632,"kb_avail":93738916,"statfs":{"total":95995035648,"available":95988649984,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5767168},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Mar 26 20:54:18 2024","interfaces":[{"interface":"back","average":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.47299999999999998,"5min":0.40699999999999997,"15min":0.39600000000000002},"min":{"1min":0.308,"5min":0.25,"15min":0.25},"max":{"1min":0.96699999999999997,"5min":0.96699999999999997,"15min":0.96699999999999997},"last":0.371}]},{"osd":1,"last update":"Tue Mar 26 20:54:18 2024","interfaces":[{"interface":"back","average":{"1min":0.42699999999999999,"5min":0.42199999999999999,"15min":0.42099999999999999},"min":{"1min":0.311,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"last":0.35499999999999998},{"interface":"front","average":{"1min":0.42699999999999999,"5min":0.39500000000000002,"15min":0.39000000000000001},"min":{"1min":0.30299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":0.67500000000000004,"5min":0.67500000000000004,"15min":0.67500000000000004},"last":0.39700000000000002}]},{"osd":3,"last update":"Tue Mar 26 20:54:46 2024","interfaces":[{"interface":"back","average":{"1min":0.41699999999999998,"5min":0.44500000000000001,"15min":0.44900000000000001},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":0.41099999999999998},{"interface":"front","average":{"1min":0.47699999999999998,"5min":0.48399999999999999,"15min":0.48499999999999999},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.41699999999999998}]},{"osd":4,"last update":"Tue Mar 26 20:55:09 2024","interfaces":[{"interface":"back","average":{"1min":0.628,"5min":0.58799999999999997,"15min":0.58199999999999996},"min":{"1min":0.49399999999999999,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.54300000000000004},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.58599999999999997,"15min":0.58199999999999996},"min":{"1min":0.47699999999999998,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.98599999999999999,"5min":0.98599999999999999,"15min":0.98599999999999999},"last":0.51400000000000001}]},{"osd":5,"last update":"Tue Mar 26 20:54:27 2024","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.58399999999999996},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.48299999999999998}]},{"osd":6,"last update":"Tue Mar 26 20:54:54 2024","interfaces":[{"interface":"back","average":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"min":{"1min":0.5,"5min":0.5,"15min":0.5},"max":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.497},{"interface":"front","average":{"1min":0.66600000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"min":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"max":{"1min":0.94599999999999995,"5min":0.94599999999999995,"15min":0.94599999999999995},"last":0.55700000000000005}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.50900000000000001}]}]},{"osd":3,"up_from":26,"seq":111669149730,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5720,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739432,"statfs":{"total":95995035648,"available":95989178368,"internally_reserved":0,"allocated":606208,"data_stored":244968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Mar 26 20:54:43 2024","interfaces":[{"interface":"back","average":{"1min":0.37,"5min":0.39800000000000002,"15min":0.40300000000000002},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.56499999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"last":0.38400000000000001},{"interface":"front","average":{"1min":0.38500000000000001,"5min":0.41799999999999998,"15min":0.42399999999999999},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":0.47999999999999998,"5min":0.55100000000000005,"15min":0.55100000000000005},"last":0.39500000000000002}]},{"osd":1,"last update":"Tue Mar 26 20:54:43 2024","interfaces":[{"interface":"back","average":{"1min":0.42099999999999999,"5min":0.44,"15min":0.443},"min":{"1min":0.32000000000000001,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.58399999999999996,"5min":0.61399999999999999,"15min":0.61399999999999999},"last":0.438},{"interface":"front","average":{"1min":0.39100000000000001,"5min":0.43099999999999999,"15min":0.438},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.54200000000000004,"5min":0.60099999999999998,"15min":0.60099999999999998},"last":0.35799999999999998}]},{"osd":2,"last update":"Tue Mar 26 20:54:43 2024","interfaces":[{"interface":"back","average":{"1min":0.47499999999999998,"5min":0.48499999999999999,"15min":0.48699999999999999},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.58399999999999996,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.45600000000000002,"5min":0.47499999999999998,"15min":0.47799999999999998},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.58999999999999997,"5min":0.65300000000000002,"15min":0.65300000000000002},"last":0.41599999999999998}]},{"osd":4,"last update":"Tue Mar 26 20:55:06 2024","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.61199999999999999,"15min":0.61599999999999999},"min":{"1min":0.45000000000000001,"5min":0.432,"15min":0.432},"max":{"1min":0.73099999999999998,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.66800000000000004},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.61799999999999999,"15min":0.62},"min":{"1min":0.51400000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.68200000000000005,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.61299999999999999}]},{"osd":5,"last update":"Tue Mar 26 20:54:31 2024","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":1.1990000000000001,"5min":1.1990000000000001,"15min":1.1990000000000001},"last":0.58299999999999996},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.627}]},{"osd":6,"last update":"Tue Mar 26 20:54:53 2024","interfaces":[{"interface":"back","average":{"1min":1.0209999999999999,"5min":1.0209999999999999,"15min":1.0209999999999999},"min":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":6.4699999999999998,"5min":6.4699999999999998,"15min":6.4699999999999998},"last":0.65700000000000003},{"interface":"front","average":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"min":{"1min":0.504,"5min":0.504,"15min":0.504},"max":{"1min":0.81399999999999995,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.56999999999999995}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.59199999999999997}]}]},{"osd":4,"up_from":31,"seq":133143986204,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5784,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739368,"statfs":{"total":95995035648,"available":95989112832,"internally_reserved":0,"allocated":606208,"data_stored":244910,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Mar 26 20:54:04 2024","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.48699999999999999,"15min":0.48699999999999999},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.73999999999999999},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":2.6819999999999999,"5min":2.6819999999999999,"15min":2.6819999999999999},"last":0.61199999999999999}]},{"osd":1,"last update":"Tue Mar 26 20:54:04 2024","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.62},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.68400000000000005}]},{"osd":2,"last update":"Tue Mar 26 20:54:04 2024","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.629,"5min":0.629,"15min":0.629},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":2.7280000000000002,"5min":2.7280000000000002,"15min":2.7280000000000002},"last":0.66200000000000003}]},{"osd":3,"last update":"Tue Mar 26 20:54:04 2024","interfaces":[{"interface":"back","average":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":2.7109999999999999,"5min":2.7109999999999999,"15min":2.7109999999999999},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":2.653,"5min":2.653,"15min":2.653},"last":0.67300000000000004}]},{"osd":5,"last update":"Tue Mar 26 20:54:29 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":2.6379999999999999,"5min":2.6379999999999999,"15min":2.6379999999999999},"last":0.751},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.97199999999999998,"5min":0.97199999999999998,"15min":0.97199999999999998},"last":0.70799999999999996}]},{"osd":6,"last update":"Tue Mar 26 20:54:55 2024","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":1.0109999999999999,"5min":1.0109999999999999,"15min":1.0109999999999999},"last":0.65000000000000002},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.98699999999999999,"5min":0.98699999999999999,"15min":0.98699999999999999},"last":0.79500000000000004}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.76500000000000001}]}]},{"osd":5,"up_from":39,"seq":167503724568,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5656,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739496,"statfs":{"total":95995035648,"available":95989243904,"internally_reserved":0,"allocated":606208,"data_stored":244968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Mar 26 20:54:29 2024","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.501,"15min":0.501},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.61299999999999999},{"interface":"front","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.59399999999999997}]},{"osd":1,"last update":"Tue Mar 26 20:54:29 2024","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.73299999999999998},{"interface":"front","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.64900000000000002}]},{"osd":2,"last update":"Tue Mar 26 20:54:29 2024","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.65800000000000003},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.69399999999999995}]},{"osd":3,"last update":"Tue Mar 26 20:54:29 2024","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.749},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.83099999999999996,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.63600000000000001}]},{"osd":4,"last update":"Tue Mar 26 20:54:29 2024","interfaces":[{"interface":"back","average":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.629,"5min":0.629,"15min":0.629},"last":0.625},{"interface":"front","average":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.68400000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.57599999999999996}]},{"osd":6,"last update":"Tue Mar 26 20:54:51 2024","interfaces":[{"interface":"back","average":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.54600000000000004}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.60299999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-26T20:55:13.087 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-03-26T20:55:13.087 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-03-26T20:55:13.087 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-03-26T20:55:13.087 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-26T20:55:13.306 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/mon.smithi063/config 2024-03-26T20:55:13.782 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:13 smithi063 ceph-mon[24818]: from='client.14442 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-26T20:55:14.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:13 smithi174 ceph-mon[26934]: from='client.14442 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-26T20:55:14.672 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:14 smithi063 ceph-mon[24818]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-26T20:55:15.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:14 smithi174 ceph-mon[26934]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-26T20:55:15.115 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-03-26T20:55:15.115 INFO:teuthology.orchestra.run.smithi063.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-03-26T20:55:15.791 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-03-26T20:55:15.791 INFO:tasks.cephadm:Setup complete, yielding 2024-03-26T20:55:15.792 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-03-26T20:55:15.803 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi063.front.sepia.ceph.com 2024-03-26T20:55:15.803 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -- bash -c 'ceph orch status' 2024-03-26T20:55:16.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:15 smithi174 ceph-mon[26934]: from='client.? 172.21.15.63:0/3561877385' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-26T20:55:16.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:15 smithi063 ceph-mon[24818]: from='client.? 172.21.15.63:0/3561877385' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-26T20:55:16.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:16 smithi063 ceph-mon[24818]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-26T20:55:17.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:16 smithi174 ceph-mon[26934]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-26T20:55:17.790 INFO:teuthology.orchestra.run.smithi063.stdout:Backend: cephadm 2024-03-26T20:55:17.790 INFO:teuthology.orchestra.run.smithi063.stdout:Available: Yes 2024-03-26T20:55:17.790 INFO:teuthology.orchestra.run.smithi063.stdout:Paused: No 2024-03-26T20:55:18.294 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -- bash -c 'ceph orch ps' 2024-03-26T20:55:18.713 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:18 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T20:55:18.713 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:18 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:55:18.713 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:18 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T20:55:18.713 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:18 smithi063 ceph-mon[24818]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-26T20:55:18.713 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:18 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:55:18.714 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:18 smithi063 ceph-mon[24818]: from='client.14450 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:55:18.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:18 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T20:55:18.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:18 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:55:18.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:18 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T20:55:18.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:18 smithi174 ceph-mon[26934]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-26T20:55:18.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:18 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:55:18.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:18 smithi174 ceph-mon[26934]: from='client.14450 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:55:20.196 INFO:teuthology.orchestra.run.smithi063.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-26T20:55:20.196 INFO:teuthology.orchestra.run.smithi063.stdout:alertmanager.smithi063 smithi063 *:9093,9094 running (4m) 2m ago 5m 15.4M - ba2b418f427c f01f1c52094d 2024-03-26T20:55:20.196 INFO:teuthology.orchestra.run.smithi063.stdout:crash.smithi063 smithi063 running (5m) 2m ago 5m 6409k - 17.2.0 e1d6a67b021e 706c62f8af89 2024-03-26T20:55:20.196 INFO:teuthology.orchestra.run.smithi063.stdout:crash.smithi174 smithi174 running (5m) 65s ago 5m 6409k - 17.2.0 e1d6a67b021e 2c6bfe6d0ca6 2024-03-26T20:55:20.196 INFO:teuthology.orchestra.run.smithi063.stdout:grafana.smithi063 smithi063 *:3000 running (4m) 2m ago 5m 44.6M - 8.3.5 dad864ee21e9 74e19c64e1c4 2024-03-26T20:55:20.196 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.smithi063.lxhujt smithi063 *:9283 running (6m) 2m ago 6m 447M - 17.2.0 e1d6a67b021e 903406c7bde7 2024-03-26T20:55:20.196 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.smithi174.sexoin smithi174 *:8443,9283 running (5m) 65s ago 4m 413M - 17.2.0 e1d6a67b021e ccd5e4d16e93 2024-03-26T20:55:20.196 INFO:teuthology.orchestra.run.smithi063.stdout:mon.smithi063 smithi063 running (6m) 2m ago 7m 42.3M 2048M 17.2.0 e1d6a67b021e e6bd58b6c0f9 2024-03-26T20:55:20.196 INFO:teuthology.orchestra.run.smithi063.stdout:mon.smithi174 smithi174 running (4m) 65s ago 4m 39.5M 2048M 17.2.0 e1d6a67b021e 0f81c09f0c0a 2024-03-26T20:55:20.196 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.smithi063 smithi063 *:9100 running (5m) 2m ago 5m 16.4M - 1dbe0e931976 17d79ef02892 2024-03-26T20:55:20.196 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.smithi174 smithi174 *:9100 running (4m) 65s ago 4m 17.5M - 1dbe0e931976 6582440a94dc 2024-03-26T20:55:20.197 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0 smithi063 running (4m) 2m ago 4m 34.4M 3216M 17.2.0 e1d6a67b021e 761b1388d8fa 2024-03-26T20:55:20.197 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1 smithi063 running (3m) 2m ago 3m 33.8M 3216M 17.2.0 e1d6a67b021e 108275c17cad 2024-03-26T20:55:20.197 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2 smithi063 running (3m) 2m ago 3m 30.4M 3216M 17.2.0 e1d6a67b021e 107f685ad08d 2024-03-26T20:55:20.197 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3 smithi063 running (2m) 2m ago 2m 13.4M 3216M 17.2.0 e1d6a67b021e ab0f1a855957 2024-03-26T20:55:20.197 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4 smithi174 running (2m) 65s ago 2m 32.7M 3984M 17.2.0 e1d6a67b021e a0e877ecc06d 2024-03-26T20:55:20.197 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5 smithi174 running (119s) 65s ago 118s 35.7M 3984M 17.2.0 e1d6a67b021e 378c2aca7e71 2024-03-26T20:55:20.197 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6 smithi174 running (95s) 65s ago 95s 32.9M 3984M 17.2.0 e1d6a67b021e ffbb5b7ad3b6 2024-03-26T20:55:20.197 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7 smithi174 running (71s) 65s ago 70s 9.94M 3984M 17.2.0 e1d6a67b021e c1d57fcabb3f 2024-03-26T20:55:20.197 INFO:teuthology.orchestra.run.smithi063.stdout:prometheus.smithi063 smithi063 *:9095 running (4m) 2m ago 4m 54.1M - 514e6a882f6e a93dcc31497d 2024-03-26T20:55:20.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:20 smithi063 ceph-mon[24818]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-26T20:55:20.867 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -- bash -c 'ceph orch ls' 2024-03-26T20:55:21.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:20 smithi174 ceph-mon[26934]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-26T20:55:21.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:21 smithi063 ceph-mon[24818]: from='client.14454 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:55:22.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:21 smithi174 ceph-mon[26934]: from='client.14454 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:55:22.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:22 smithi063 ceph-mon[24818]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-26T20:55:22.937 INFO:teuthology.orchestra.run.smithi063.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-03-26T20:55:22.937 INFO:teuthology.orchestra.run.smithi063.stdout:alertmanager ?:9093,9094 1/1 2m ago 6m count:1 2024-03-26T20:55:22.937 INFO:teuthology.orchestra.run.smithi063.stdout:crash 2/2 2m ago 6m * 2024-03-26T20:55:22.937 INFO:teuthology.orchestra.run.smithi063.stdout:grafana ?:3000 1/1 2m ago 6m count:1 2024-03-26T20:55:22.937 INFO:teuthology.orchestra.run.smithi063.stdout:mgr 2/2 2m ago 6m count:2 2024-03-26T20:55:22.937 INFO:teuthology.orchestra.run.smithi063.stdout:mon 2/2 2m ago 5m smithi063:172.21.15.63=smithi063;smithi174:172.21.15.174=smithi174;count:2 2024-03-26T20:55:22.937 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter ?:9100 2/2 2m ago 6m * 2024-03-26T20:55:22.937 INFO:teuthology.orchestra.run.smithi063.stdout:osd 8 2m ago - 2024-03-26T20:55:22.937 INFO:teuthology.orchestra.run.smithi063.stdout:prometheus ?:9095 1/1 2m ago 6m count:1 2024-03-26T20:55:23.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:22 smithi174 ceph-mon[26934]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-26T20:55:23.543 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -- bash -c 'ceph orch host ls' 2024-03-26T20:55:23.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:23 smithi063 ceph-mon[24818]: from='client.14458 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:55:24.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:23 smithi174 ceph-mon[26934]: from='client.14458 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:55:24.826 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:24 smithi063 ceph-mon[24818]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-26T20:55:24.949 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:24 smithi174 ceph-mon[26934]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-26T20:55:25.477 INFO:teuthology.orchestra.run.smithi063.stdout:HOST ADDR LABELS STATUS 2024-03-26T20:55:25.477 INFO:teuthology.orchestra.run.smithi063.stdout:smithi063 172.21.15.63 2024-03-26T20:55:25.477 INFO:teuthology.orchestra.run.smithi063.stdout:smithi174 172.21.15.174 2024-03-26T20:55:25.477 INFO:teuthology.orchestra.run.smithi063.stdout:2 hosts in cluster 2024-03-26T20:55:26.073 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -- bash -c 'ceph orch device ls' 2024-03-26T20:55:26.694 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:26 smithi063 ceph-mon[24818]: from='client.14462 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:55:26.694 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:26 smithi063 ceph-mon[24818]: pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-26T20:55:27.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:26 smithi174 ceph-mon[26934]: from='client.14462 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:55:27.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:26 smithi174 ceph-mon[26934]: pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-26T20:55:28.050 INFO:teuthology.orchestra.run.smithi063.stdout:HOST PATH TYPE DEVICE ID SIZE AVAILABLE REJECT REASONS 2024-03-26T20:55:28.051 INFO:teuthology.orchestra.run.smithi063.stdout:smithi063 /dev/nvme0n1 ssd INTEL_SSDPEDMD400G4_CVFT5344005B400BGN 400G LVM detected, locked 2024-03-26T20:55:28.051 INFO:teuthology.orchestra.run.smithi063.stdout:smithi174 /dev/nvme0n1 ssd INTEL_SSDPEDMD400G4_PHFT62050074400BGN 400G LVM detected, locked 2024-03-26T20:55:28.619 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:28 smithi063 ceph-mon[24818]: pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-26T20:55:28.643 INFO:teuthology.run_tasks:Running task vip.exec... 2024-03-26T20:55:28.655 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi063.front.sepia.ceph.com 2024-03-26T20:55:28.655 DEBUG:teuthology.orchestra.run.smithi063:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'systemctl stop nfs-server' 2024-03-26T20:55:28.683 INFO:teuthology.orchestra.run.smithi063.stderr:+ systemctl stop nfs-server 2024-03-26T20:55:28.690 INFO:tasks.vip:Running commands on role host.b host ubuntu@smithi174.front.sepia.ceph.com 2024-03-26T20:55:28.690 DEBUG:teuthology.orchestra.run.smithi174:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'systemctl stop nfs-server' 2024-03-26T20:55:28.719 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:28 smithi174 ceph-mon[26934]: pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-26T20:55:28.722 INFO:teuthology.orchestra.run.smithi174.stderr:+ systemctl stop nfs-server 2024-03-26T20:55:28.730 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-03-26T20:55:28.739 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi063.front.sepia.ceph.com 2024-03-26T20:55:28.739 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -- bash -c 'ceph fs volume create foofs' 2024-03-26T20:55:29.734 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:29 smithi063 ceph-mon[24818]: from='client.14466 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:55:29.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:29 smithi174 ceph-mon[26934]: from='client.14466 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:55:30.551 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:30 smithi063 ceph-mon[24818]: pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-26T20:55:30.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:30 smithi174 ceph-mon[26934]: pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-26T20:55:31.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:31 smithi174 ceph-mon[26934]: from='client.14470 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foofs", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:55:31.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:31 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]: dispatch 2024-03-26T20:55:31.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:31 smithi063 ceph-mon[24818]: from='client.14470 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foofs", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:55:31.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:31 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]: dispatch 2024-03-26T20:55:32.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:32 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]': finished 2024-03-26T20:55:32.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:32 smithi174 ceph-mon[26934]: osdmap e60: 8 total, 8 up, 8 in 2024-03-26T20:55:32.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:32 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]: dispatch 2024-03-26T20:55:32.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:32 smithi174 ceph-mon[26934]: pgmap v210: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-26T20:55:32.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:32 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]': finished 2024-03-26T20:55:32.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:32 smithi063 ceph-mon[24818]: osdmap e60: 8 total, 8 up, 8 in 2024-03-26T20:55:32.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:32 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]: dispatch 2024-03-26T20:55:32.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:32 smithi063 ceph-mon[24818]: pgmap v210: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-26T20:55:32.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:32 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-mon-smithi063[24814]: 2024-03-26T20:55:32.396+0000 7fe3755e3700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-26T20:55:33.551 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:33 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]': finished 2024-03-26T20:55:33.551 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:33 smithi063 ceph-mon[24818]: osdmap e61: 8 total, 8 up, 8 in 2024-03-26T20:55:33.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:33 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]: dispatch 2024-03-26T20:55:33.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:33 smithi063 ceph-mon[24818]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-26T20:55:33.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:33 smithi063 ceph-mon[24818]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-03-26T20:55:33.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:33 smithi063 ceph-mon[24818]: osdmap e62: 8 total, 8 up, 8 in 2024-03-26T20:55:33.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:33 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]': finished 2024-03-26T20:55:33.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:33 smithi063 ceph-mon[24818]: fsmap foofs:0 2024-03-26T20:55:33.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:33 smithi063 ceph-mon[24818]: Saving service mds.foofs spec with placement count:2 2024-03-26T20:55:33.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:33 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:55:33.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:33 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T20:55:33.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:33 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:55:33.553 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:33 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T20:55:33.553 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:33 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:55:33.553 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:33 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:55:33.553 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:33 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi063.tzaznz", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-26T20:55:33.553 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:33 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi063.tzaznz", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-03-26T20:55:33.553 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:33 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:55:33.553 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:33 smithi063 ceph-mon[24818]: Deploying daemon mds.foofs.smithi063.tzaznz on smithi063 2024-03-26T20:55:33.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:33 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]': finished 2024-03-26T20:55:33.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:33 smithi174 ceph-mon[26934]: osdmap e61: 8 total, 8 up, 8 in 2024-03-26T20:55:33.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:33 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]: dispatch 2024-03-26T20:55:33.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:33 smithi174 ceph-mon[26934]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-26T20:55:33.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:33 smithi174 ceph-mon[26934]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-03-26T20:55:33.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:33 smithi174 ceph-mon[26934]: osdmap e62: 8 total, 8 up, 8 in 2024-03-26T20:55:33.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:33 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]': finished 2024-03-26T20:55:33.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:33 smithi174 ceph-mon[26934]: fsmap foofs:0 2024-03-26T20:55:33.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:33 smithi174 ceph-mon[26934]: Saving service mds.foofs spec with placement count:2 2024-03-26T20:55:33.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:33 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:55:33.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:33 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T20:55:33.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:33 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:55:33.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:33 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T20:55:33.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:33 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:55:33.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:33 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:55:33.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:33 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi063.tzaznz", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-26T20:55:33.800 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:33 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi063.tzaznz", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-03-26T20:55:33.800 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:33 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:55:33.800 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:33 smithi174 ceph-mon[26934]: Deploying daemon mds.foofs.smithi063.tzaznz on smithi063 2024-03-26T20:55:34.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:34 smithi063 ceph-mon[24818]: osdmap e63: 8 total, 8 up, 8 in 2024-03-26T20:55:34.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:34 smithi063 ceph-mon[24818]: pgmap v214: 65 pgs: 6 creating+peering, 35 unknown, 24 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-26T20:55:34.593 INFO:teuthology.run_tasks:Running task cephadm.wait_for_service... 2024-03-26T20:55:34.602 INFO:tasks.cephadm:Waiting for ceph service mds.foofs to start (timeout 300)... 2024-03-26T20:55:34.602 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -- ceph orch ls -f json 2024-03-26T20:55:34.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:34 smithi174 ceph-mon[26934]: osdmap e63: 8 total, 8 up, 8 in 2024-03-26T20:55:34.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:34 smithi174 ceph-mon[26934]: pgmap v214: 65 pgs: 6 creating+peering, 35 unknown, 24 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-26T20:55:36.252 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:36 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:55:36.252 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:36 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi174.gitdvk", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-26T20:55:36.253 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:36 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi174.gitdvk", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-03-26T20:55:36.253 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:36 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:55:36.253 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:36 smithi063 ceph-mon[24818]: Deploying daemon mds.foofs.smithi174.gitdvk on smithi174 2024-03-26T20:55:36.253 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:36 smithi063 ceph-mon[24818]: mds.? [v2:172.21.15.63:6834/1380786213,v1:172.21.15.63:6835/1380786213] up:boot 2024-03-26T20:55:36.253 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:36 smithi063 ceph-mon[24818]: daemon mds.foofs.smithi063.tzaznz assigned to filesystem foofs as rank 0 (now has 1 ranks) 2024-03-26T20:55:36.253 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:36 smithi063 ceph-mon[24818]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-03-26T20:55:36.253 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:36 smithi063 ceph-mon[24818]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-03-26T20:55:36.253 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:36 smithi063 ceph-mon[24818]: Cluster is now healthy 2024-03-26T20:55:36.253 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:36 smithi063 ceph-mon[24818]: fsmap foofs:0 1 up:standby 2024-03-26T20:55:36.253 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:36 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi063.tzaznz"}]: dispatch 2024-03-26T20:55:36.253 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:36 smithi063 ceph-mon[24818]: fsmap foofs:1 {0=foofs.smithi063.tzaznz=up:creating} 2024-03-26T20:55:36.253 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:36 smithi063 ceph-mon[24818]: daemon mds.foofs.smithi063.tzaznz is now active in filesystem foofs as rank 0 2024-03-26T20:55:36.253 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:36 smithi063 ceph-mon[24818]: pgmap v215: 65 pgs: 6 creating+peering, 24 unknown, 35 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-26T20:55:36.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:36 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:55:36.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:36 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi174.gitdvk", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-26T20:55:36.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:36 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi174.gitdvk", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-03-26T20:55:36.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:36 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:55:36.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:36 smithi174 ceph-mon[26934]: Deploying daemon mds.foofs.smithi174.gitdvk on smithi174 2024-03-26T20:55:36.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:36 smithi174 ceph-mon[26934]: mds.? [v2:172.21.15.63:6834/1380786213,v1:172.21.15.63:6835/1380786213] up:boot 2024-03-26T20:55:36.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:36 smithi174 ceph-mon[26934]: daemon mds.foofs.smithi063.tzaznz assigned to filesystem foofs as rank 0 (now has 1 ranks) 2024-03-26T20:55:36.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:36 smithi174 ceph-mon[26934]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-03-26T20:55:36.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:36 smithi174 ceph-mon[26934]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-03-26T20:55:36.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:36 smithi174 ceph-mon[26934]: Cluster is now healthy 2024-03-26T20:55:36.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:36 smithi174 ceph-mon[26934]: fsmap foofs:0 1 up:standby 2024-03-26T20:55:36.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:36 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi063.tzaznz"}]: dispatch 2024-03-26T20:55:36.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:36 smithi174 ceph-mon[26934]: fsmap foofs:1 {0=foofs.smithi063.tzaznz=up:creating} 2024-03-26T20:55:36.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:36 smithi174 ceph-mon[26934]: daemon mds.foofs.smithi063.tzaznz is now active in filesystem foofs as rank 0 2024-03-26T20:55:36.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:36 smithi174 ceph-mon[26934]: pgmap v215: 65 pgs: 6 creating+peering, 24 unknown, 35 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-03-26T20:55:36.944 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-03-26T20:55:36.945 INFO:teuthology.orchestra.run.smithi063.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-03-26T20:49:01.786722Z", "last_refresh": "2024-03-26T20:52:42.063512Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-03-26T20:48:56.123811Z", "last_refresh": "2024-03-26T20:52:42.063583Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-03-26T20:48:58.928241Z", "last_refresh": "2024-03-26T20:52:42.063662Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-03-26T20:55:32.416024Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-03-26T20:55:32.409168Z", "running": 0, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-03-26T20:48:54.137895Z", "last_refresh": "2024-03-26T20:52:42.063436Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi063:172.21.15.63=smithi063", "smithi174:172.21.15.174=smithi174"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-03-26T20:49:47.043046Z", "last_refresh": "2024-03-26T20:52:42.063289Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-03-26T20:49:00.204434Z", "last_refresh": "2024-03-26T20:52:42.063733Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-03-26T20:52:42.063865Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-03-26T20:48:57.277368Z", "last_refresh": "2024-03-26T20:52:42.063799Z", "ports": [9095], "running": 1, "size": 1}}] 2024-03-26T20:55:37.549 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:37 smithi063 ceph-mon[24818]: mds.? [v2:172.21.15.63:6834/1380786213,v1:172.21.15.63:6835/1380786213] up:active 2024-03-26T20:55:37.549 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:37 smithi063 ceph-mon[24818]: fsmap foofs:1 {0=foofs.smithi063.tzaznz=up:active} 2024-03-26T20:55:37.549 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:37 smithi063 ceph-mon[24818]: from='client.14478 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-26T20:55:37.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:37 smithi174 ceph-mon[26934]: mds.? [v2:172.21.15.63:6834/1380786213,v1:172.21.15.63:6835/1380786213] up:active 2024-03-26T20:55:37.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:37 smithi174 ceph-mon[26934]: fsmap foofs:1 {0=foofs.smithi063.tzaznz=up:active} 2024-03-26T20:55:37.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:37 smithi174 ceph-mon[26934]: from='client.14478 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-26T20:55:37.550 INFO:tasks.cephadm:mds.foofs has 0/2 2024-03-26T20:55:38.552 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -- ceph orch ls -f json 2024-03-26T20:55:38.965 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:38 smithi063 ceph-mon[24818]: pgmap v216: 65 pgs: 65 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 341 B/s wr, 1 op/s 2024-03-26T20:55:38.965 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:38 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:55:38.965 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:38 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T20:55:38.966 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:38 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:55:38.966 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:38 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T20:55:38.966 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:38 smithi063 ceph-mon[24818]: mds.? [v2:172.21.15.174:6832/2611248035,v1:172.21.15.174:6833/2611248035] up:boot 2024-03-26T20:55:38.966 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:38 smithi063 ceph-mon[24818]: fsmap foofs:1 {0=foofs.smithi063.tzaznz=up:active} 1 up:standby 2024-03-26T20:55:38.966 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:38 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi174.gitdvk"}]: dispatch 2024-03-26T20:55:38.966 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:38 smithi063 ceph-mon[24818]: fsmap foofs:1 {0=foofs.smithi063.tzaznz=up:active} 1 up:standby 2024-03-26T20:55:39.028 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:38 smithi174 ceph-mon[26934]: pgmap v216: 65 pgs: 65 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 341 B/s wr, 1 op/s 2024-03-26T20:55:39.028 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:38 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:55:39.028 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:38 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T20:55:39.028 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:38 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:55:39.028 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:38 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T20:55:39.028 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:38 smithi174 ceph-mon[26934]: mds.? [v2:172.21.15.174:6832/2611248035,v1:172.21.15.174:6833/2611248035] up:boot 2024-03-26T20:55:39.029 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:38 smithi174 ceph-mon[26934]: fsmap foofs:1 {0=foofs.smithi063.tzaznz=up:active} 1 up:standby 2024-03-26T20:55:39.029 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:38 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi174.gitdvk"}]: dispatch 2024-03-26T20:55:39.029 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:38 smithi174 ceph-mon[26934]: fsmap foofs:1 {0=foofs.smithi063.tzaznz=up:active} 1 up:standby 2024-03-26T20:55:40.644 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-03-26T20:55:40.644 INFO:teuthology.orchestra.run.smithi063.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-03-26T20:49:01.786722Z", "last_refresh": "2024-03-26T20:52:42.063512Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-03-26T20:48:56.123811Z", "last_refresh": "2024-03-26T20:52:42.063583Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-03-26T20:48:58.928241Z", "last_refresh": "2024-03-26T20:52:42.063662Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-03-26T20:55:32.416024Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-03-26T20:55:32.409168Z", "last_refresh": "2024-03-26T20:55:39.682018Z", "running": 1, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-03-26T20:48:54.137895Z", "last_refresh": "2024-03-26T20:52:42.063436Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi063:172.21.15.63=smithi063", "smithi174:172.21.15.174=smithi174"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-03-26T20:49:47.043046Z", "last_refresh": "2024-03-26T20:52:42.063289Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-03-26T20:49:00.204434Z", "last_refresh": "2024-03-26T20:52:42.063733Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-03-26T20:52:42.063865Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-03-26T20:48:57.277368Z", "last_refresh": "2024-03-26T20:52:42.063799Z", "ports": [9095], "running": 1, "size": 1}}] 2024-03-26T20:55:41.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:40 smithi174 ceph-mon[26934]: pgmap v217: 65 pgs: 65 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s wr, 2 op/s 2024-03-26T20:55:41.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:40 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:55:41.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:40 smithi063 ceph-mon[24818]: pgmap v217: 65 pgs: 65 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s wr, 2 op/s 2024-03-26T20:55:41.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:40 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:55:41.550 INFO:tasks.cephadm:mds.foofs has 1/2 2024-03-26T20:55:41.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:41 smithi063 ceph-mon[24818]: from='client.14484 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-26T20:55:42.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:41 smithi174 ceph-mon[26934]: from='client.14484 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-26T20:55:42.551 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -- ceph orch ls -f json 2024-03-26T20:55:42.831 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:42 smithi063 ceph-mon[24818]: pgmap v218: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s wr, 4 op/s 2024-03-26T20:55:42.831 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:42 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:55:43.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:42 smithi174 ceph-mon[26934]: pgmap v218: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s wr, 4 op/s 2024-03-26T20:55:43.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:42 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:55:45.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:44 smithi174 ceph-mon[26934]: pgmap v219: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-03-26T20:55:45.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:44 smithi063 ceph-mon[24818]: pgmap v219: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-03-26T20:55:45.726 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-03-26T20:55:45.726 INFO:teuthology.orchestra.run.smithi063.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-03-26T20:49:01.786722Z", "last_refresh": "2024-03-26T20:55:45.707435Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-03-26T20:48:56.123811Z", "last_refresh": "2024-03-26T20:55:39.681302Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-03-26T20:48:58.928241Z", "last_refresh": "2024-03-26T20:55:45.707574Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-03-26T20:55:32.416024Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-03-26T20:55:32.409168Z", "last_refresh": "2024-03-26T20:55:39.682018Z", "running": 2, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-03-26T20:48:54.137895Z", "last_refresh": "2024-03-26T20:55:39.681466Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi063:172.21.15.63=smithi063", "smithi174:172.21.15.174=smithi174"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-03-26T20:49:47.043046Z", "last_refresh": "2024-03-26T20:55:39.681554Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-03-26T20:49:00.204434Z", "last_refresh": "2024-03-26T20:55:39.681642Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-03-26T20:55:39.681724Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-03-26T20:48:57.277368Z", "last_refresh": "2024-03-26T20:55:45.707725Z", "ports": [9095], "running": 1, "size": 1}}] 2024-03-26T20:55:46.536 INFO:tasks.cephadm:mds.foofs has 2/2 2024-03-26T20:55:46.536 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-03-26T20:55:46.545 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi063.front.sepia.ceph.com 2024-03-26T20:55:46.545 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -- bash -c 'ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2' 2024-03-26T20:55:46.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:46 smithi063 ceph-mon[24818]: pgmap v220: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-03-26T20:55:46.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:46 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:55:46.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:46 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:55:46.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:46 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T20:55:46.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:46 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:55:46.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:46 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T20:55:46.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:46 smithi063 ceph-mon[24818]: from='client.14488 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-26T20:55:46.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:46 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:55:47.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:46 smithi174 ceph-mon[26934]: pgmap v220: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-03-26T20:55:47.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:46 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:55:47.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:46 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:55:47.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:46 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T20:55:47.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:46 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:55:47.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:46 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T20:55:47.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:46 smithi174 ceph-mon[26934]: from='client.14488 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-26T20:55:47.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:46 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:55:48.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:48 smithi174 ceph-mon[26934]: pgmap v221: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-03-26T20:55:48.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:48 smithi063 ceph-mon[24818]: pgmap v221: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-03-26T20:55:49.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:49 smithi174 ceph-mon[26934]: from='client.14492 -' entity='client.admin' cmd=[{"prefix": "nfs cluster create", "cluster_id": "foo", "placement": "2", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:55:49.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:49 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd pool create", "pool": ".nfs"}]: dispatch 2024-03-26T20:55:49.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:49 smithi063 ceph-mon[24818]: from='client.14492 -' entity='client.admin' cmd=[{"prefix": "nfs cluster create", "cluster_id": "foo", "placement": "2", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:55:49.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:49 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd pool create", "pool": ".nfs"}]: dispatch 2024-03-26T20:55:50.412 INFO:teuthology.orchestra.run.smithi063.stdout:NFS Cluster Created Successfully 2024-03-26T20:55:50.551 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:50 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "osd pool create", "pool": ".nfs"}]': finished 2024-03-26T20:55:50.551 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:50 smithi063 ceph-mon[24818]: osdmap e64: 8 total, 8 up, 8 in 2024-03-26T20:55:50.551 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:50 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2024-03-26T20:55:50.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:50 smithi063 ceph-mon[24818]: pgmap v223: 97 pgs: 32 unknown, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 307 B/s wr, 2 op/s 2024-03-26T20:55:50.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:50 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "osd pool create", "pool": ".nfs"}]': finished 2024-03-26T20:55:50.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:50 smithi174 ceph-mon[26934]: osdmap e64: 8 total, 8 up, 8 in 2024-03-26T20:55:50.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:50 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2024-03-26T20:55:50.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:50 smithi174 ceph-mon[26934]: pgmap v223: 97 pgs: 32 unknown, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 307 B/s wr, 2 op/s 2024-03-26T20:55:51.071 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -- bash -c 'ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake' 2024-03-26T20:55:51.709 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:51 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]': finished 2024-03-26T20:55:51.710 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:51 smithi063 ceph-mon[24818]: osdmap e65: 8 total, 8 up, 8 in 2024-03-26T20:55:51.710 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:51 smithi063 ceph-mon[24818]: Saving service nfs.foo spec with placement count:2 2024-03-26T20:55:51.710 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:51 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:55:51.710 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:51 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T20:55:51.710 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:51 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:55:51.710 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:51 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T20:55:51.710 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:51 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:55:51.710 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:51 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:55:51.710 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:51 smithi063 ceph-mon[24818]: Creating key for client.nfs.foo.0.0.smithi063.fgmpwa 2024-03-26T20:55:51.710 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:51 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi063.fgmpwa", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-03-26T20:55:51.711 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:51 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi063.fgmpwa", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-03-26T20:55:51.711 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:51 smithi063 ceph-mon[24818]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-03-26T20:55:51.711 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:51 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-03-26T20:55:51.711 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:51 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-03-26T20:55:51.711 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:51 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:55:51.711 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:51 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-03-26T20:55:51.711 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:51 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-03-26T20:55:51.711 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:51 smithi063 ceph-mon[24818]: Rados config object exists: conf-nfs.foo 2024-03-26T20:55:51.711 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:51 smithi063 ceph-mon[24818]: Creating key for client.nfs.foo.0.0.smithi063.fgmpwa-rgw 2024-03-26T20:55:51.711 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:51 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi063.fgmpwa-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-03-26T20:55:51.711 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:51 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi063.fgmpwa-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-03-26T20:55:51.712 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:51 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:55:51.712 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:51 smithi063 ceph-mon[24818]: Deploying daemon nfs.foo.0.0.smithi063.fgmpwa on smithi063 2024-03-26T20:55:51.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:51 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]': finished 2024-03-26T20:55:51.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:51 smithi174 ceph-mon[26934]: osdmap e65: 8 total, 8 up, 8 in 2024-03-26T20:55:51.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:51 smithi174 ceph-mon[26934]: Saving service nfs.foo spec with placement count:2 2024-03-26T20:55:51.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:51 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:55:51.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:51 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T20:55:51.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:51 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:55:51.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:51 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T20:55:51.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:51 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:55:51.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:51 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:55:51.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:51 smithi174 ceph-mon[26934]: Creating key for client.nfs.foo.0.0.smithi063.fgmpwa 2024-03-26T20:55:51.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:51 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi063.fgmpwa", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-03-26T20:55:51.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:51 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi063.fgmpwa", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-03-26T20:55:51.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:51 smithi174 ceph-mon[26934]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-03-26T20:55:51.800 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:51 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-03-26T20:55:51.800 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:51 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-03-26T20:55:51.800 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:51 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:55:51.800 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:51 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-03-26T20:55:51.800 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:51 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-03-26T20:55:51.800 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:51 smithi174 ceph-mon[26934]: Rados config object exists: conf-nfs.foo 2024-03-26T20:55:51.800 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:51 smithi174 ceph-mon[26934]: Creating key for client.nfs.foo.0.0.smithi063.fgmpwa-rgw 2024-03-26T20:55:51.800 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:51 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi063.fgmpwa-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-03-26T20:55:51.800 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:51 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi063.fgmpwa-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-03-26T20:55:51.800 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:51 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:55:51.800 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:51 smithi174 ceph-mon[26934]: Deploying daemon nfs.foo.0.0.smithi063.fgmpwa on smithi063 2024-03-26T20:55:52.641 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:52 smithi063 ceph-mon[24818]: osdmap e66: 8 total, 8 up, 8 in 2024-03-26T20:55:52.641 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:52 smithi063 ceph-mon[24818]: mgrmap e20: smithi063.lxhujt(active, since 5m), standbys: smithi174.sexoin 2024-03-26T20:55:52.641 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:52 smithi063 ceph-mon[24818]: pgmap v226: 97 pgs: 4 creating+peering, 16 unknown, 77 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-03-26T20:55:52.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:52 smithi174 ceph-mon[26934]: osdmap e66: 8 total, 8 up, 8 in 2024-03-26T20:55:52.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:52 smithi174 ceph-mon[26934]: mgrmap e20: smithi063.lxhujt(active, since 5m), standbys: smithi174.sexoin 2024-03-26T20:55:52.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:52 smithi174 ceph-mon[26934]: pgmap v226: 97 pgs: 4 creating+peering, 16 unknown, 77 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-03-26T20:55:54.769 INFO:teuthology.orchestra.run.smithi063.stderr:Invalid command: Unexpected argument '--clusterid' 2024-03-26T20:55:54.769 INFO:teuthology.orchestra.run.smithi063.stderr:nfs export create cephfs [] [--readonly] [--client_addr ...] [--squash ] : Create a CephFS export 2024-03-26T20:55:54.770 INFO:teuthology.orchestra.run.smithi063.stderr:Error EINVAL: invalid command 2024-03-26T20:55:54.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:54 smithi063 ceph-mon[24818]: pgmap v227: 97 pgs: 4 creating+peering, 93 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-03-26T20:55:54.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:54 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:55:54.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:54 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi174.pytptt", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-03-26T20:55:54.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:54 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi174.pytptt", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-03-26T20:55:54.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:54 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-03-26T20:55:54.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:54 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-03-26T20:55:54.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:54 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:55:54.845 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:54 smithi174 ceph-mon[26934]: pgmap v227: 97 pgs: 4 creating+peering, 93 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-03-26T20:55:54.846 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:54 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:55:54.846 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:54 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi174.pytptt", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-03-26T20:55:54.846 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:54 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi174.pytptt", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-03-26T20:55:54.846 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:54 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-03-26T20:55:54.846 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:54 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-03-26T20:55:54.846 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:54 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:55:55.110 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-03-26T20:55:55.110 INFO:teuthology.orchestra.run.smithi063.stdout: "bind": "/fake", 2024-03-26T20:55:55.110 INFO:teuthology.orchestra.run.smithi063.stdout: "fs": "foofs", 2024-03-26T20:55:55.110 INFO:teuthology.orchestra.run.smithi063.stdout: "path": "/", 2024-03-26T20:55:55.110 INFO:teuthology.orchestra.run.smithi063.stdout: "cluster": "foo", 2024-03-26T20:55:55.110 INFO:teuthology.orchestra.run.smithi063.stdout: "mode": "RW" 2024-03-26T20:55:55.110 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-03-26T20:55:55.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:55 smithi063 ceph-mon[24818]: Creating key for client.nfs.foo.1.0.smithi174.pytptt 2024-03-26T20:55:55.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:55 smithi063 ceph-mon[24818]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-03-26T20:55:55.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:55 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-03-26T20:55:55.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:55 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-03-26T20:55:55.857 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -- bash -c 'while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done' 2024-03-26T20:55:56.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:55 smithi174 ceph-mon[26934]: Creating key for client.nfs.foo.1.0.smithi174.pytptt 2024-03-26T20:55:56.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:55 smithi174 ceph-mon[26934]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-03-26T20:55:56.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:55 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-03-26T20:55:56.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:55 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-03-26T20:55:56.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:56 smithi063 ceph-mon[24818]: from='client.14526 -' entity='client.admin' cmd=[{"prefix": "nfs export create cephfs", "fsname": "foofs", "cluster_id": "foo", "pseudo_path": "/fake", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:55:56.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:56 smithi063 ceph-mon[24818]: pgmap v228: 97 pgs: 97 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 166 B/s rd, 500 B/s wr, 0 op/s 2024-03-26T20:55:57.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:56 smithi174 ceph-mon[26934]: from='client.14526 -' entity='client.admin' cmd=[{"prefix": "nfs export create cephfs", "fsname": "foofs", "cluster_id": "foo", "pseudo_path": "/fake", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:55:57.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:56 smithi174 ceph-mon[26934]: pgmap v228: 97 pgs: 97 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 166 B/s rd, 500 B/s wr, 0 op/s 2024-03-26T20:55:57.747 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:57 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-03-26T20:55:57.747 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:57 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-03-26T20:55:57.747 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:57 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi174.pytptt-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-03-26T20:55:57.747 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:57 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi174.pytptt-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-03-26T20:55:57.747 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:57 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:55:57.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:57 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-03-26T20:55:57.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:57 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-03-26T20:55:57.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:57 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi174.pytptt-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-03-26T20:55:57.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:57 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi174.pytptt-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-03-26T20:55:57.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:57 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:55:58.879 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:58 smithi174 ceph-mon[26934]: Rados config object exists: conf-nfs.foo 2024-03-26T20:55:58.879 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:58 smithi174 ceph-mon[26934]: Creating key for client.nfs.foo.1.0.smithi174.pytptt-rgw 2024-03-26T20:55:58.879 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:58 smithi174 ceph-mon[26934]: Deploying daemon nfs.foo.1.0.smithi174.pytptt on smithi174 2024-03-26T20:55:58.879 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:58 smithi174 ceph-mon[26934]: pgmap v229: 97 pgs: 97 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1.4 KiB/s wr, 3 op/s 2024-03-26T20:55:59.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:58 smithi063 ceph-mon[24818]: Rados config object exists: conf-nfs.foo 2024-03-26T20:55:59.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:58 smithi063 ceph-mon[24818]: Creating key for client.nfs.foo.1.0.smithi174.pytptt-rgw 2024-03-26T20:55:59.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:58 smithi063 ceph-mon[24818]: Deploying daemon nfs.foo.1.0.smithi174.pytptt on smithi174 2024-03-26T20:55:59.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:58 smithi063 ceph-mon[24818]: pgmap v229: 97 pgs: 97 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1.4 KiB/s wr, 3 op/s 2024-03-26T20:55:59.862 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:55:59 smithi063 ceph-mon[24818]: from='client.14540 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:55:59.863 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:55:59 smithi174 ceph-mon[26934]: from='client.14540 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:56:01.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:56:00 smithi174 ceph-mon[26934]: from='client.14544 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:56:01.235 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:56:00 smithi174 ceph-mon[26934]: pgmap v230: 97 pgs: 97 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.2 KiB/s wr, 2 op/s 2024-03-26T20:56:01.235 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:56:00 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:56:01.236 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:56:00 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T20:56:01.236 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:56:00 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:56:01.236 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:56:00 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T20:56:01.236 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:56:00 smithi063 ceph-mon[24818]: from='client.14544 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:56:01.236 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:56:00 smithi063 ceph-mon[24818]: pgmap v230: 97 pgs: 97 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.2 KiB/s wr, 2 op/s 2024-03-26T20:56:01.236 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:56:00 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:56:01.237 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:56:00 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T20:56:01.237 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:56:00 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:56:01.237 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:56:00 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T20:56:02.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:56:01 smithi174 ceph-mon[26934]: from='client.14560 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:56:02.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:56:01 smithi063 ceph-mon[24818]: from='client.14560 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:56:02.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:56:02 smithi063 ceph-mon[24818]: pgmap v231: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 4.0 KiB/s rd, 2.5 KiB/s wr, 7 op/s 2024-03-26T20:56:02.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:56:02 smithi063 ceph-mon[24818]: from='client.14564 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:56:02.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:56:02 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:56:02.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:56:02 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:56:03.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:56:02 smithi174 ceph-mon[26934]: pgmap v231: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 4.0 KiB/s rd, 2.5 KiB/s wr, 7 op/s 2024-03-26T20:56:03.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:56:02 smithi174 ceph-mon[26934]: from='client.14564 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:56:03.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:56:02 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:56:03.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:56:02 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:56:04.561 INFO:teuthology.orchestra.run.smithi063.stdout:nfs.foo ?:2049 2/2 1s ago 14s count:2 2024-03-26T20:56:05.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:56:04 smithi174 ceph-mon[26934]: from='client.14568 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:56:05.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:56:04 smithi174 ceph-mon[26934]: pgmap v232: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 2.1 KiB/s wr, 5 op/s 2024-03-26T20:56:05.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:56:04 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:56:05.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:56:04 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:56:05.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:56:04 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:56:05.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:56:04 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T20:56:05.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:56:04 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:56:05.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:56:04 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T20:56:05.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:56:04 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:56:05.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:56:04 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:56:05.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:56:04 smithi063 ceph-mon[24818]: from='client.14568 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:56:05.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:56:04 smithi063 ceph-mon[24818]: pgmap v232: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 2.1 KiB/s wr, 5 op/s 2024-03-26T20:56:05.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:56:04 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:56:05.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:56:04 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:56:05.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:56:04 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:56:05.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:56:04 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T20:56:05.052 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:56:04 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:56:05.052 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:56:04 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T20:56:05.052 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:56:04 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:56:05.052 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:56:04 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:56:05.295 INFO:teuthology.run_tasks:Running task vip.exec... 2024-03-26T20:56:05.306 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi063.front.sepia.ceph.com 2024-03-26T20:56:05.306 DEBUG:teuthology.orchestra.run.smithi063:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'mkdir /mnt/foo' 2024-03-26T20:56:05.334 INFO:teuthology.orchestra.run.smithi063.stderr:+ mkdir /mnt/foo 2024-03-26T20:56:05.341 DEBUG:teuthology.orchestra.run.smithi063:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done' 2024-03-26T20:56:05.408 INFO:teuthology.orchestra.run.smithi063.stderr:++ hostname 2024-03-26T20:56:05.409 INFO:teuthology.orchestra.run.smithi063.stderr:+ mount -t nfs smithi063:/fake /mnt/foo -o sync 2024-03-26T20:56:05.817 DEBUG:teuthology.orchestra.run.smithi063:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'echo test > /mnt/foo/testfile' 2024-03-26T20:56:05.884 INFO:teuthology.orchestra.run.smithi063.stderr:+ echo test 2024-03-26T20:56:05.894 DEBUG:teuthology.orchestra.run.smithi063:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c sync 2024-03-26T20:56:05.960 INFO:teuthology.orchestra.run.smithi063.stderr:+ sync 2024-03-26T20:56:06.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:56:05 smithi174 ceph-mon[26934]: from='client.14572 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:56:06.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:56:05 smithi063 ceph-mon[24818]: from='client.14572 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:56:06.193 INFO:teuthology.run_tasks:Running task parallel... 2024-03-26T20:56:06.216 INFO:teuthology.task.parallel:starting parallel... 2024-03-26T20:56:06.217 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-03-26T20:56:06.225 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-03-26T20:56:06.225 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi063.front.sepia.ceph.com 2024-03-26T20:56:06.225 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -e sha1=42fc52b86b14d4f88c40d75eb7102e0ab3918f55 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-03-26T20:56:06.226 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-03-26T20:56:06.226 INFO:teuthology.task.sequential:In sequential, running task exec... 2024-03-26T20:56:06.227 INFO:teuthology.task.exec:Executing custom commands... 2024-03-26T20:56:06.227 INFO:teuthology.task.exec:Running commands on role host.a host ubuntu@smithi063.front.sepia.ceph.com 2024-03-26T20:56:06.227 DEBUG:teuthology.orchestra.run.smithi063:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'cd /mnt/foo && dbench 5 -t 600 || true' 2024-03-26T20:56:06.322 INFO:teuthology.orchestra.run.smithi063.stdout:dbench version 4.00 - Copyright Andrew Tridgell 1999-2004 2024-03-26T20:56:06.322 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-03-26T20:56:06.322 INFO:teuthology.orchestra.run.smithi063.stdout:Running for 600 seconds with load '/usr/share/dbench/client.txt' and minimum warmup 120 secs 2024-03-26T20:56:06.323 INFO:teuthology.orchestra.run.smithi063.stdout:3 of 5 processes prepared for launch 0 sec 2024-03-26T20:56:06.423 INFO:teuthology.orchestra.run.smithi063.stdout:5 of 5 processes prepared for launch 0 sec 2024-03-26T20:56:06.424 INFO:teuthology.orchestra.run.smithi063.stdout:releasing clients 2024-03-26T20:56:07.022 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:56:06 smithi063 ceph-mon[24818]: pgmap v233: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 2.1 KiB/s wr, 5 op/s 2024-03-26T20:56:07.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:56:06 smithi174 ceph-mon[26934]: pgmap v233: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 2.1 KiB/s wr, 5 op/s 2024-03-26T20:56:07.423 INFO:teuthology.orchestra.run.smithi063.stdout: 5 138 29.85 MB/sec warmup 1 sec latency 38.478 ms 2024-03-26T20:56:08.424 INFO:teuthology.orchestra.run.smithi063.stdout: 5 274 25.66 MB/sec warmup 2 sec latency 33.068 ms 2024-03-26T20:56:08.635 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:56:08 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.3", "id": [7, 4]}]: dispatch 2024-03-26T20:56:08.635 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:56:08 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2024-03-26T20:56:08.635 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:56:08 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 5]}]: dispatch 2024-03-26T20:56:08.635 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:56:08 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2024-03-26T20:56:08.635 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:56:08 smithi063 ceph-mon[24818]: pgmap v234: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 2.0 KiB/s wr, 5 op/s 2024-03-26T20:56:08.635 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:56:08 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi063.lxhujt/trash_purge_schedule"}]: dispatch 2024-03-26T20:56:08.635 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:56:08 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi063.lxhujt/mirror_snapshot_schedule"}]: dispatch 2024-03-26T20:56:08.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:56:08 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.3", "id": [7, 4]}]: dispatch 2024-03-26T20:56:08.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:56:08 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2024-03-26T20:56:08.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:56:08 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 5]}]: dispatch 2024-03-26T20:56:08.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:56:08 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2024-03-26T20:56:08.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:56:08 smithi174 ceph-mon[26934]: pgmap v234: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 2.0 KiB/s wr, 5 op/s 2024-03-26T20:56:08.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:56:08 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi063.lxhujt/trash_purge_schedule"}]: dispatch 2024-03-26T20:56:08.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:56:08 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi063.lxhujt/mirror_snapshot_schedule"}]: dispatch 2024-03-26T20:56:09.413 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:56:09 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.3", "id": [7, 4]}]': finished 2024-03-26T20:56:09.414 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:56:09 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]': finished 2024-03-26T20:56:09.414 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:56:09 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 5]}]': finished 2024-03-26T20:56:09.414 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:56:09 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]': finished 2024-03-26T20:56:09.414 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:56:09 smithi063 ceph-mon[24818]: osdmap e67: 8 total, 8 up, 8 in 2024-03-26T20:56:09.424 INFO:teuthology.orchestra.run.smithi063.stdout: 5 406 23.95 MB/sec warmup 3 sec latency 32.076 ms 2024-03-26T20:56:09.670 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -e sha1=42fc52b86b14d4f88c40d75eb7102e0ab3918f55 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-03-26T20:56:09.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:56:09 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.3", "id": [7, 4]}]': finished 2024-03-26T20:56:09.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:56:09 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]': finished 2024-03-26T20:56:09.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:56:09 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 5]}]': finished 2024-03-26T20:56:09.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:56:09 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]': finished 2024-03-26T20:56:09.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:56:09 smithi174 ceph-mon[26934]: osdmap e67: 8 total, 8 up, 8 in 2024-03-26T20:56:10.424 INFO:teuthology.orchestra.run.smithi063.stdout: 5 551 23.78 MB/sec warmup 4 sec latency 36.279 ms 2024-03-26T20:56:10.469 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:56:10 smithi063 ceph-mon[24818]: osdmap e68: 8 total, 8 up, 8 in 2024-03-26T20:56:10.469 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:56:10 smithi063 ceph-mon[24818]: pgmap v237: 97 pgs: 97 active+clean; 15 MiB data, 62 MiB used, 715 GiB / 715 GiB avail; 1.9 MiB/s wr, 33 op/s 2024-03-26T20:56:10.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:56:10 smithi174 ceph-mon[26934]: osdmap e68: 8 total, 8 up, 8 in 2024-03-26T20:56:10.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:56:10 smithi174 ceph-mon[26934]: pgmap v237: 97 pgs: 97 active+clean; 15 MiB data, 62 MiB used, 715 GiB / 715 GiB avail; 1.9 MiB/s wr, 33 op/s 2024-03-26T20:56:11.424 INFO:teuthology.orchestra.run.smithi063.stdout: 5 818 21.49 MB/sec warmup 5 sec latency 30.634 ms 2024-03-26T20:56:12.424 INFO:teuthology.orchestra.run.smithi063.stdout: 5 1162 19.18 MB/sec warmup 6 sec latency 53.432 ms 2024-03-26T20:56:12.827 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:56:12 smithi063 ceph-mon[24818]: pgmap v238: 97 pgs: 97 active+clean; 60 MiB data, 121 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 7.7 MiB/s wr, 442 op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-03-26T20:56:13.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:56:12 smithi174 ceph-mon[26934]: pgmap v238: 97 pgs: 97 active+clean; 60 MiB data, 121 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 7.7 MiB/s wr, 442 op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-03-26T20:56:13.424 INFO:teuthology.orchestra.run.smithi063.stdout: 5 2064 19.00 MB/sec warmup 7 sec latency 21.142 ms 2024-03-26T20:56:13.630 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -e sha1=42fc52b86b14d4f88c40d75eb7102e0ab3918f55 -- bash -c 'ceph config set global log_to_journald false --force' 2024-03-26T20:56:14.424 INFO:teuthology.orchestra.run.smithi063.stdout: 5 2866 20.28 MB/sec warmup 8 sec latency 18.814 ms 2024-03-26T20:56:14.585 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:56:14 smithi063 ceph-mon[24818]: pgmap v239: 97 pgs: 97 active+clean; 109 MiB data, 158 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 14 MiB/s wr, 602 op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-03-26T20:56:15.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:56:14 smithi174 ceph-mon[26934]: pgmap v239: 97 pgs: 97 active+clean; 109 MiB data, 158 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 14 MiB/s wr, 602 op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-03-26T20:56:15.426 INFO:teuthology.orchestra.run.smithi063.stdout: 5 3697 21.53 MB/sec warmup 9 sec latency 22.982 ms 2024-03-26T20:56:16.428 INFO:teuthology.orchestra.run.smithi063.stdout: 5 4402 20.31 MB/sec warmup 10 sec latency 12.447 ms 2024-03-26T20:56:16.595 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:56:16 smithi063 ceph-mon[24818]: pgmap v240: 97 pgs: 97 active+clean; 109 MiB data, 165 MiB used, 715 GiB / 715 GiB avail; 2.2 KiB/s rd, 15 MiB/s wr, 613 op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-03-26T20:56:17.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:56:16 smithi174 ceph-mon[26934]: pgmap v240: 97 pgs: 97 active+clean; 109 MiB data, 165 MiB used, 715 GiB / 715 GiB avail; 2.2 KiB/s rd, 15 MiB/s wr, 613 op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-03-26T20:56:17.428 INFO:teuthology.orchestra.run.smithi063.stdout: 5 5231 20.59 MB/sec warmup 11 sec latency 137.877 ms 2024-03-26T20:56:18.428 INFO:teuthology.orchestra.run.smithi063.stdout: 5 6089 19.92 MB/sec warmup 12 sec latency 25.023 ms 2024-03-26T20:56:18.472 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:56:18 smithi063 ceph-mon[24818]: pgmap v241: 97 pgs: 97 active+clean; 123 MiB data, 184 MiB used, 715 GiB / 715 GiB avail; 2.0 KiB/s rd, 15 MiB/s wr, 785 op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-03-26T20:56:18.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:56:18 smithi174 ceph-mon[26934]: pgmap v241: 97 pgs: 97 active+clean; 123 MiB data, 184 MiB used, 715 GiB / 715 GiB avail; 2.0 KiB/s rd, 15 MiB/s wr, 785 op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-03-26T20:56:18.949 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -e sha1=42fc52b86b14d4f88c40d75eb7102e0ab3918f55 -- bash -c 'ceph mgr module enable nfs --force' 2024-03-26T20:56:19.428 INFO:teuthology.orchestra.run.smithi063.stdout: 5 7166 22.18 MB/sec warmup 13 sec latency 16.872 ms 2024-03-26T20:56:20.428 INFO:teuthology.orchestra.run.smithi063.stdout: 5 7882 21.29 MB/sec warmup 14 sec latency 12.134 ms 2024-03-26T20:56:20.620 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:56:20 smithi063 ceph-mon[24818]: pgmap v242: 97 pgs: 97 active+clean; 129 MiB data, 239 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 15 MiB/s wr, 887 op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-03-26T20:56:21.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:56:20 smithi174 ceph-mon[26934]: pgmap v242: 97 pgs: 97 active+clean; 129 MiB data, 239 MiB used, 715 GiB / 715 GiB avail; 1.8 KiB/s rd, 15 MiB/s wr, 887 op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-03-26T20:56:21.428 INFO:teuthology.orchestra.run.smithi063.stdout: 5 8800 21.48 MB/sec warmup 15 sec latency 20.808 ms 2024-03-26T20:56:22.428 INFO:teuthology.orchestra.run.smithi063.stdout: 5 9614 20.93 MB/sec warmup 16 sec latency 24.315 ms 2024-03-26T20:56:22.723 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:56:22 smithi063 ceph-mon[24818]: pgmap v243: 97 pgs: 97 active+clean; 137 MiB data, 287 MiB used, 715 GiB / 715 GiB avail; 7.8 KiB/s rd, 15 MiB/s wr, 990 op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-03-26T20:56:23.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:56:22 smithi174 ceph-mon[26934]: pgmap v243: 97 pgs: 97 active+clean; 137 MiB data, 287 MiB used, 715 GiB / 715 GiB avail; 7.8 KiB/s rd, 15 MiB/s wr, 990 op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-03-26T20:56:23.428 INFO:teuthology.orchestra.run.smithi063.stdout: 5 10720 22.63 MB/sec warmup 17 sec latency 11.596 ms 2024-03-26T20:56:23.608 INFO:teuthology.orchestra.run.smithi063.stderr:module 'nfs' is already enabled 2024-03-26T20:56:23.730 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:56:23 smithi063 ceph-mon[24818]: from='client.? 172.21.15.63:0/1711275419' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-03-26T20:56:24.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:56:23 smithi174 ceph-mon[26934]: from='client.? 172.21.15.63:0/1711275419' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-03-26T20:56:24.428 INFO:teuthology.orchestra.run.smithi063.stdout: 5 11357 21.81 MB/sec warmup 18 sec latency 13.560 ms 2024-03-26T20:56:24.734 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:56:24 smithi063 ceph-mon[24818]: pgmap v244: 97 pgs: 97 active+clean; 146 MiB data, 363 MiB used, 715 GiB / 715 GiB avail; 7.7 KiB/s rd, 14 MiB/s wr, 1k op/s 2024-03-26T20:56:24.735 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:56:24 smithi063 ceph-mon[24818]: from='client.? 172.21.15.63:0/1711275419' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "nfs", "force": true}]': finished 2024-03-26T20:56:24.735 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:56:24 smithi063 ceph-mon[24818]: mgrmap e21: smithi063.lxhujt(active, since 6m), standbys: smithi174.sexoin 2024-03-26T20:56:24.776 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -e sha1=42fc52b86b14d4f88c40d75eb7102e0ab3918f55 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-03-26T20:56:24.949 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:56:24 smithi174 ceph-mon[26934]: pgmap v244: 97 pgs: 97 active+clean; 146 MiB data, 363 MiB used, 715 GiB / 715 GiB avail; 7.7 KiB/s rd, 14 MiB/s wr, 1k op/s 2024-03-26T20:56:24.949 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:56:24 smithi174 ceph-mon[26934]: from='client.? 172.21.15.63:0/1711275419' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "nfs", "force": true}]': finished 2024-03-26T20:56:24.950 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:56:24 smithi174 ceph-mon[26934]: mgrmap e21: smithi063.lxhujt(active, since 6m), standbys: smithi174.sexoin 2024-03-26T20:56:25.428 INFO:teuthology.orchestra.run.smithi063.stdout: 5 12214 21.96 MB/sec warmup 19 sec latency 14.140 ms 2024-03-26T20:56:26.432 INFO:teuthology.orchestra.run.smithi063.stdout: 5 12983 21.28 MB/sec warmup 20 sec latency 28.999 ms 2024-03-26T20:56:26.742 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:56:26 smithi063 ceph-mon[24818]: pgmap v245: 97 pgs: 97 active+clean; 147 MiB data, 375 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 10 MiB/s wr, 907 op/s 2024-03-26T20:56:27.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:56:26 smithi174 ceph-mon[26934]: pgmap v245: 97 pgs: 97 active+clean; 147 MiB data, 375 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 10 MiB/s wr, 907 op/s 2024-03-26T20:56:27.432 INFO:teuthology.orchestra.run.smithi063.stdout: 5 13758 21.75 MB/sec warmup 21 sec latency 29.428 ms 2024-03-26T20:56:28.251 INFO:teuthology.orchestra.run.smithi063.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:42fc52b86b14d4f88c40d75eb7102e0ab3918f55 2024-03-26T20:56:28.432 INFO:teuthology.orchestra.run.smithi063.stdout: 5 14477 22.08 MB/sec warmup 22 sec latency 19.621 ms 2024-03-26T20:56:28.502 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:56:28 smithi063 ceph-mon[24818]: pgmap v246: 97 pgs: 97 active+clean; 149 MiB data, 418 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 12 MiB/s wr, 1.08k op/s 2024-03-26T20:56:28.502 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:56:28 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:56:28.502 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:56:28 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T20:56:28.502 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:56:28 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:56:28.502 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:56:28 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T20:56:28.612 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:56:28 smithi174 ceph-mon[26934]: pgmap v246: 97 pgs: 97 active+clean; 149 MiB data, 418 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 12 MiB/s wr, 1.08k op/s 2024-03-26T20:56:28.612 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:56:28 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:56:28.612 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:56:28 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T20:56:28.612 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:56:28 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:56:28.612 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:56:28 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T20:56:29.432 INFO:teuthology.orchestra.run.smithi063.stdout: 5 15046 21.30 MB/sec warmup 23 sec latency 31.777 ms 2024-03-26T20:56:29.520 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-03-26T20:56:29.521 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi063.front.sepia.ceph.com 2024-03-26T20:56:29.521 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -e sha1=42fc52b86b14d4f88c40d75eb7102e0ab3918f55 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done' 2024-03-26T20:56:30.015 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:56:29 smithi063 ceph-mon[24818]: from='client.14592 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:42fc52b86b14d4f88c40d75eb7102e0ab3918f55", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:56:30.015 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:56:29 smithi063 ceph-mon[24818]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:42fc52b86b14d4f88c40d75eb7102e0ab3918f55 2024-03-26T20:56:30.015 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:56:29 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:56:30.015 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:56:29 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:56:30.015 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:56:29 smithi063 ceph-mon[24818]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:42fc52b86b14d4f88c40d75eb7102e0ab3918f55 2024-03-26T20:56:30.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:56:29 smithi174 ceph-mon[26934]: from='client.14592 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:42fc52b86b14d4f88c40d75eb7102e0ab3918f55", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:56:30.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:56:29 smithi174 ceph-mon[26934]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:42fc52b86b14d4f88c40d75eb7102e0ab3918f55 2024-03-26T20:56:30.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:56:29 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:56:30.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:56:29 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:56:30.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:56:29 smithi174 ceph-mon[26934]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:42fc52b86b14d4f88c40d75eb7102e0ab3918f55 2024-03-26T20:56:30.433 INFO:teuthology.orchestra.run.smithi063.stdout: 5 15792 21.37 MB/sec warmup 24 sec latency 20.886 ms 2024-03-26T20:56:31.018 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:56:30 smithi063 ceph-mon[24818]: pgmap v247: 97 pgs: 97 active+clean; 158 MiB data, 439 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 12 MiB/s wr, 1.12k op/s 2024-03-26T20:56:31.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:56:30 smithi174 ceph-mon[26934]: pgmap v247: 97 pgs: 97 active+clean; 158 MiB data, 439 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 12 MiB/s wr, 1.12k op/s 2024-03-26T20:56:31.432 INFO:teuthology.orchestra.run.smithi063.stdout: 5 16664 20.95 MB/sec warmup 25 sec latency 19.057 ms 2024-03-26T20:56:32.433 INFO:teuthology.orchestra.run.smithi063.stdout: 5 17645 21.75 MB/sec warmup 26 sec latency 15.328 ms 2024-03-26T20:56:33.031 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:56:32 smithi063 ceph-mon[24818]: pgmap v248: 97 pgs: 97 active+clean; 156 MiB data, 498 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 11 MiB/s wr, 1.02k op/s 2024-03-26T20:56:33.211 INFO:teuthology.orchestra.run.smithi063.stdout:true 2024-03-26T20:56:33.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:56:32 smithi174 ceph-mon[26934]: pgmap v248: 97 pgs: 97 active+clean; 156 MiB data, 498 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 11 MiB/s wr, 1.02k op/s 2024-03-26T20:56:33.433 INFO:teuthology.orchestra.run.smithi063.stdout: 5 18132 21.58 MB/sec warmup 27 sec latency 21.630 ms 2024-03-26T20:56:34.322 INFO:teuthology.orchestra.run.smithi063.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-26T20:56:34.322 INFO:teuthology.orchestra.run.smithi063.stdout:alertmanager.smithi063 smithi063 *:9093,9094 running (5m) 29s ago 6m 22.9M - ba2b418f427c f01f1c52094d 2024-03-26T20:56:34.322 INFO:teuthology.orchestra.run.smithi063.stdout:crash.smithi063 smithi063 running (6m) 29s ago 6m 6409k - 17.2.0 e1d6a67b021e 706c62f8af89 2024-03-26T20:56:34.322 INFO:teuthology.orchestra.run.smithi063.stdout:crash.smithi174 smithi174 running (6m) 31s ago 6m 6409k - 17.2.0 e1d6a67b021e 2c6bfe6d0ca6 2024-03-26T20:56:34.323 INFO:teuthology.orchestra.run.smithi063.stdout:grafana.smithi063 smithi063 *:3000 running (5m) 29s ago 6m 51.8M - 8.3.5 dad864ee21e9 74e19c64e1c4 2024-03-26T20:56:34.323 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foofs.smithi063.tzaznz smithi063 running (59s) 29s ago 59s 14.9M - 17.2.0 e1d6a67b021e 10a989fc7954 2024-03-26T20:56:34.323 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foofs.smithi174.gitdvk smithi174 running (57s) 31s ago 56s 11.1M - 17.2.0 e1d6a67b021e 4339a5d98d1d 2024-03-26T20:56:34.323 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.smithi063.lxhujt smithi063 *:9283 running (8m) 29s ago 8m 459M - 17.2.0 e1d6a67b021e 903406c7bde7 2024-03-26T20:56:34.323 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.smithi174.sexoin smithi174 *:8443,9283 running (6m) 31s ago 6m 413M - 17.2.0 e1d6a67b021e ccd5e4d16e93 2024-03-26T20:56:34.323 INFO:teuthology.orchestra.run.smithi063.stdout:mon.smithi063 smithi063 running (8m) 29s ago 8m 47.4M 2048M 17.2.0 e1d6a67b021e e6bd58b6c0f9 2024-03-26T20:56:34.323 INFO:teuthology.orchestra.run.smithi063.stdout:mon.smithi174 smithi174 running (6m) 31s ago 6m 37.3M 2048M 17.2.0 e1d6a67b021e 0f81c09f0c0a 2024-03-26T20:56:34.323 INFO:teuthology.orchestra.run.smithi063.stdout:nfs.foo.0.0.smithi063.fgmpwa smithi063 *:2049 running (40s) 29s ago 40s 47.0M - 4.0 e1d6a67b021e 3d6c0dd9ce51 2024-03-26T20:56:34.323 INFO:teuthology.orchestra.run.smithi063.stdout:nfs.foo.1.0.smithi174.pytptt smithi174 *:2049 running (35s) 31s ago 34s 48.7M - 4.0 e1d6a67b021e f220dbb449a3 2024-03-26T20:56:34.323 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.smithi063 smithi063 *:9100 running (6m) 29s ago 6m 15.9M - 1dbe0e931976 17d79ef02892 2024-03-26T20:56:34.323 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.smithi174 smithi174 *:9100 running (6m) 31s ago 6m 20.1M - 1dbe0e931976 6582440a94dc 2024-03-26T20:56:34.323 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0 smithi063 running (5m) 29s ago 5m 42.7M 3216M 17.2.0 e1d6a67b021e 761b1388d8fa 2024-03-26T20:56:34.323 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1 smithi063 running (4m) 29s ago 4m 42.2M 3216M 17.2.0 e1d6a67b021e 108275c17cad 2024-03-26T20:56:34.323 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2 smithi063 running (4m) 29s ago 4m 40.3M 3216M 17.2.0 e1d6a67b021e 107f685ad08d 2024-03-26T20:56:34.323 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3 smithi063 running (3m) 29s ago 3m 42.7M 3216M 17.2.0 e1d6a67b021e ab0f1a855957 2024-03-26T20:56:34.324 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4 smithi174 running (3m) 31s ago 3m 39.1M 3984M 17.2.0 e1d6a67b021e a0e877ecc06d 2024-03-26T20:56:34.324 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5 smithi174 running (3m) 31s ago 3m 41.7M 3984M 17.2.0 e1d6a67b021e 378c2aca7e71 2024-03-26T20:56:34.324 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6 smithi174 running (2m) 31s ago 2m 42.0M 3984M 17.2.0 e1d6a67b021e ffbb5b7ad3b6 2024-03-26T20:56:34.324 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7 smithi174 running (2m) 31s ago 2m 38.9M 3984M 17.2.0 e1d6a67b021e c1d57fcabb3f 2024-03-26T20:56:34.324 INFO:teuthology.orchestra.run.smithi063.stdout:prometheus.smithi063 smithi063 *:9095 running (5m) 29s ago 5m 60.4M - 514e6a882f6e a93dcc31497d 2024-03-26T20:56:34.433 INFO:teuthology.orchestra.run.smithi063.stdout: 5 18641 20.96 MB/sec warmup 28 sec latency 33.280 ms 2024-03-26T20:56:35.009 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-03-26T20:56:35.009 INFO:teuthology.orchestra.run.smithi063.stdout: "mon": { 2024-03-26T20:56:35.009 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-26T20:56:35.009 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T20:56:35.009 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr": { 2024-03-26T20:56:35.009 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-26T20:56:35.009 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T20:56:35.009 INFO:teuthology.orchestra.run.smithi063.stdout: "osd": { 2024-03-26T20:56:35.009 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-03-26T20:56:35.010 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T20:56:35.010 INFO:teuthology.orchestra.run.smithi063.stdout: "mds": { 2024-03-26T20:56:35.010 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-26T20:56:35.010 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T20:56:35.010 INFO:teuthology.orchestra.run.smithi063.stdout: "overall": { 2024-03-26T20:56:35.010 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-03-26T20:56:35.010 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-03-26T20:56:35.010 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-03-26T20:56:35.038 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:56:34 smithi063 ceph-mon[24818]: from='client.24391 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:56:35.039 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:56:34 smithi063 ceph-mon[24818]: pgmap v249: 97 pgs: 97 active+clean; 165 MiB data, 569 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 13 MiB/s wr, 1.13k op/s 2024-03-26T20:56:35.039 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:56:34 smithi063 ceph-mon[24818]: from='client.24395 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:56:35.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:56:34 smithi174 ceph-mon[26934]: from='client.24391 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:56:35.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:56:34 smithi174 ceph-mon[26934]: pgmap v249: 97 pgs: 97 active+clean; 165 MiB data, 569 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 13 MiB/s wr, 1.13k op/s 2024-03-26T20:56:35.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:56:34 smithi174 ceph-mon[26934]: from='client.24395 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:56:35.433 INFO:teuthology.orchestra.run.smithi063.stdout: 5 19229 20.97 MB/sec warmup 29 sec latency 24.452 ms 2024-03-26T20:56:35.551 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-03-26T20:56:35.551 INFO:teuthology.orchestra.run.smithi063.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:42fc52b86b14d4f88c40d75eb7102e0ab3918f55", 2024-03-26T20:56:35.551 INFO:teuthology.orchestra.run.smithi063.stdout: "in_progress": true, 2024-03-26T20:56:35.551 INFO:teuthology.orchestra.run.smithi063.stdout: "services_complete": [], 2024-03-26T20:56:35.551 INFO:teuthology.orchestra.run.smithi063.stdout: "progress": "", 2024-03-26T20:56:35.551 INFO:teuthology.orchestra.run.smithi063.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:42fc52b86b14d4f88c40d75eb7102e0ab3918f55 image" 2024-03-26T20:56:35.551 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-03-26T20:56:35.901 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:56:35 smithi063 ceph-mon[24818]: from='client.14604 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:56:35.901 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:56:35 smithi063 ceph-mon[24818]: from='client.? 172.21.15.63:0/1564583987' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T20:56:36.227 INFO:teuthology.orchestra.run.smithi063.stdout:HEALTH_OK 2024-03-26T20:56:36.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:56:35 smithi174 ceph-mon[26934]: from='client.14604 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:56:36.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:56:35 smithi174 ceph-mon[26934]: from='client.? 172.21.15.63:0/1564583987' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T20:56:36.433 INFO:teuthology.orchestra.run.smithi063.stdout: 5 19996 20.44 MB/sec warmup 30 sec latency 32.951 ms 2024-03-26T20:56:37.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:56:36 smithi174 ceph-mon[26934]: pgmap v250: 97 pgs: 97 active+clean; 165 MiB data, 578 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 9.1 MiB/s wr, 859 op/s 2024-03-26T20:56:37.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:56:36 smithi174 ceph-mon[26934]: from='client.24407 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:56:37.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:56:36 smithi174 ceph-mon[26934]: from='client.? 172.21.15.63:0/2028185452' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-26T20:56:37.049 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:56:36 smithi063 ceph-mon[24818]: pgmap v250: 97 pgs: 97 active+clean; 165 MiB data, 578 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 9.1 MiB/s wr, 859 op/s 2024-03-26T20:56:37.049 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:56:36 smithi063 ceph-mon[24818]: from='client.24407 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:56:37.049 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:56:36 smithi063 ceph-mon[24818]: from='client.? 172.21.15.63:0/2028185452' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-26T20:56:37.436 INFO:teuthology.orchestra.run.smithi063.stdout: 5 20792 20.84 MB/sec warmup 31 sec latency 15.213 ms 2024-03-26T20:56:38.436 INFO:teuthology.orchestra.run.smithi063.stdout: 5 21530 21.09 MB/sec warmup 32 sec latency 18.716 ms 2024-03-26T20:56:38.471 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:56:38 smithi063 ceph-mon[24818]: pgmap v251: 97 pgs: 97 active+clean; 171 MiB data, 627 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 11 MiB/s wr, 1.07k op/s 2024-03-26T20:56:38.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:56:38 smithi174 ceph-mon[26934]: pgmap v251: 97 pgs: 97 active+clean; 171 MiB data, 627 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 11 MiB/s wr, 1.07k op/s 2024-03-26T20:56:39.436 INFO:teuthology.orchestra.run.smithi063.stdout: 5 21988 20.59 MB/sec warmup 33 sec latency 29.950 ms 2024-03-26T20:56:40.436 INFO:teuthology.orchestra.run.smithi063.stdout: 5 22571 20.35 MB/sec warmup 34 sec latency 33.865 ms 2024-03-26T20:56:40.745 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:56:40 smithi063 ceph-mon[24818]: pgmap v252: 97 pgs: 97 active+clean; 178 MiB data, 659 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 11 MiB/s wr, 1.08k op/s 2024-03-26T20:56:41.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:56:40 smithi174 ceph-mon[26934]: pgmap v252: 97 pgs: 97 active+clean; 178 MiB data, 659 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 11 MiB/s wr, 1.08k op/s 2024-03-26T20:56:41.436 INFO:teuthology.orchestra.run.smithi063.stdout: 5 23360 20.21 MB/sec warmup 35 sec latency 20.830 ms 2024-03-26T20:56:42.439 INFO:teuthology.orchestra.run.smithi063.stdout: 5 23882 19.99 MB/sec warmup 36 sec latency 26.636 ms 2024-03-26T20:56:42.757 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:56:42 smithi063 ceph-mon[24818]: pgmap v253: 97 pgs: 97 active+clean; 183 MiB data, 682 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 9.7 MiB/s wr, 898 op/s 2024-03-26T20:56:43.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:56:42 smithi174 ceph-mon[26934]: pgmap v253: 97 pgs: 97 active+clean; 183 MiB data, 682 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 9.7 MiB/s wr, 898 op/s 2024-03-26T20:56:43.439 INFO:teuthology.orchestra.run.smithi063.stdout: 5 24506 20.23 MB/sec warmup 37 sec latency 26.878 ms 2024-03-26T20:56:44.439 INFO:teuthology.orchestra.run.smithi063.stdout: 5 25039 20.24 MB/sec warmup 38 sec latency 41.064 ms 2024-03-26T20:56:44.789 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:56:44 smithi063 ceph-mon[24818]: pgmap v254: 97 pgs: 97 active+clean; 185 MiB data, 726 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 11 MiB/s wr, 987 op/s 2024-03-26T20:56:45.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:56:44 smithi174 ceph-mon[26934]: pgmap v254: 97 pgs: 97 active+clean; 185 MiB data, 726 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 11 MiB/s wr, 987 op/s 2024-03-26T20:56:45.439 INFO:teuthology.orchestra.run.smithi063.stdout: 5 25395 19.86 MB/sec warmup 39 sec latency 29.884 ms 2024-03-26T20:56:46.439 INFO:teuthology.orchestra.run.smithi063.stdout: 5 25804 19.47 MB/sec warmup 40 sec latency 33.121 ms 2024-03-26T20:56:46.816 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:56:46 smithi063 ceph-mon[24818]: pgmap v255: 97 pgs: 97 active+clean; 187 MiB data, 751 MiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 7.9 MiB/s wr, 717 op/s 2024-03-26T20:56:47.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:56:46 smithi174 ceph-mon[26934]: pgmap v255: 97 pgs: 97 active+clean; 187 MiB data, 751 MiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 7.9 MiB/s wr, 717 op/s 2024-03-26T20:56:47.439 INFO:teuthology.orchestra.run.smithi063.stdout: 5 26233 19.28 MB/sec warmup 41 sec latency 53.581 ms 2024-03-26T20:56:48.439 INFO:teuthology.orchestra.run.smithi063.stdout: 5 26954 19.13 MB/sec warmup 42 sec latency 30.911 ms 2024-03-26T20:56:48.589 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:56:48 smithi063 ceph-mon[24818]: pgmap v256: 97 pgs: 97 active+clean; 197 MiB data, 771 MiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 9.7 MiB/s wr, 904 op/s 2024-03-26T20:56:48.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:56:48 smithi174 ceph-mon[26934]: pgmap v256: 97 pgs: 97 active+clean; 197 MiB data, 771 MiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 9.7 MiB/s wr, 904 op/s 2024-03-26T20:56:49.439 INFO:teuthology.orchestra.run.smithi063.stdout: 5 27461 18.99 MB/sec warmup 43 sec latency 46.537 ms 2024-03-26T20:56:50.447 INFO:teuthology.orchestra.run.smithi063.stdout: 5 28162 19.23 MB/sec warmup 44 sec latency 42.227 ms 2024-03-26T20:56:50.613 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:56:50 smithi063 ceph-mon[24818]: pgmap v257: 97 pgs: 97 active+clean; 198 MiB data, 790 MiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 8.6 MiB/s wr, 807 op/s 2024-03-26T20:56:51.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:56:50 smithi174 ceph-mon[26934]: pgmap v257: 97 pgs: 97 active+clean; 198 MiB data, 790 MiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 8.6 MiB/s wr, 807 op/s 2024-03-26T20:56:51.446 INFO:teuthology.orchestra.run.smithi063.stdout: 5 28770 19.30 MB/sec warmup 45 sec latency 33.496 ms 2024-03-26T20:56:52.447 INFO:teuthology.orchestra.run.smithi063.stdout: 5 29262 18.98 MB/sec warmup 46 sec latency 31.480 ms 2024-03-26T20:56:52.643 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:56:52 smithi063 ceph-mon[24818]: pgmap v258: 97 pgs: 97 active+clean; 204 MiB data, 822 MiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 8.3 MiB/s wr, 772 op/s 2024-03-26T20:56:53.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:56:52 smithi174 ceph-mon[26934]: pgmap v258: 97 pgs: 97 active+clean; 204 MiB data, 822 MiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 8.3 MiB/s wr, 772 op/s 2024-03-26T20:56:53.447 INFO:teuthology.orchestra.run.smithi063.stdout: 5 29942 18.93 MB/sec warmup 47 sec latency 19.986 ms 2024-03-26T20:56:54.447 INFO:teuthology.orchestra.run.smithi063.stdout: 5 30616 18.77 MB/sec warmup 48 sec latency 44.238 ms 2024-03-26T20:56:54.665 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:56:54 smithi063 ceph-mon[24818]: pgmap v259: 97 pgs: 97 active+clean; 208 MiB data, 897 MiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 8.8 MiB/s wr, 788 op/s 2024-03-26T20:56:54.850 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:56:54 smithi174 ceph-mon[26934]: pgmap v259: 97 pgs: 97 active+clean; 208 MiB data, 897 MiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 8.8 MiB/s wr, 788 op/s 2024-03-26T20:56:55.447 INFO:teuthology.orchestra.run.smithi063.stdout: 5 31005 18.61 MB/sec warmup 49 sec latency 34.920 ms 2024-03-26T20:56:56.447 INFO:teuthology.orchestra.run.smithi063.stdout: 5 31536 18.74 MB/sec warmup 50 sec latency 32.486 ms 2024-03-26T20:56:56.698 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:56:56 smithi063 ceph-mon[24818]: pgmap v260: 97 pgs: 97 active+clean; 214 MiB data, 907 MiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 7.8 MiB/s wr, 787 op/s 2024-03-26T20:56:57.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:56:56 smithi174 ceph-mon[26934]: pgmap v260: 97 pgs: 97 active+clean; 214 MiB data, 907 MiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 7.8 MiB/s wr, 787 op/s 2024-03-26T20:56:57.447 INFO:teuthology.orchestra.run.smithi063.stdout: 5 32117 18.83 MB/sec warmup 51 sec latency 22.307 ms 2024-03-26T20:56:58.447 INFO:teuthology.orchestra.run.smithi063.stdout: 5 32530 18.58 MB/sec warmup 52 sec latency 32.661 ms 2024-03-26T20:56:58.476 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:56:58 smithi063 ceph-mon[24818]: pgmap v261: 97 pgs: 97 active+clean; 217 MiB data, 921 MiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 8.8 MiB/s wr, 847 op/s 2024-03-26T20:56:58.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:56:58 smithi174 ceph-mon[26934]: pgmap v261: 97 pgs: 97 active+clean; 217 MiB data, 921 MiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 8.8 MiB/s wr, 847 op/s 2024-03-26T20:56:59.447 INFO:teuthology.orchestra.run.smithi063.stdout: 5 33048 18.40 MB/sec warmup 53 sec latency 24.638 ms 2024-03-26T20:57:00.447 INFO:teuthology.orchestra.run.smithi063.stdout: 5 33613 18.25 MB/sec warmup 54 sec latency 35.121 ms 2024-03-26T20:57:00.745 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:57:00 smithi063 ceph-mon[24818]: pgmap v262: 97 pgs: 97 active+clean; 216 MiB data, 933 MiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 7.9 MiB/s wr, 704 op/s 2024-03-26T20:57:01.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:57:00 smithi174 ceph-mon[26934]: pgmap v262: 97 pgs: 97 active+clean; 216 MiB data, 933 MiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 7.9 MiB/s wr, 704 op/s 2024-03-26T20:57:01.447 INFO:teuthology.orchestra.run.smithi063.stdout: 5 34362 18.25 MB/sec warmup 55 sec latency 35.813 ms 2024-03-26T20:57:02.447 INFO:teuthology.orchestra.run.smithi063.stdout: 5 35314 18.59 MB/sec warmup 56 sec latency 13.552 ms 2024-03-26T20:57:02.753 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:57:02 smithi063 ceph-mon[24818]: pgmap v263: 97 pgs: 97 active+clean; 219 MiB data, 986 MiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 7.9 MiB/s wr, 719 op/s 2024-03-26T20:57:03.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:57:02 smithi174 ceph-mon[26934]: pgmap v263: 97 pgs: 97 active+clean; 219 MiB data, 986 MiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 7.9 MiB/s wr, 719 op/s 2024-03-26T20:57:03.447 INFO:teuthology.orchestra.run.smithi063.stdout: 5 36085 18.62 MB/sec warmup 57 sec latency 16.192 ms 2024-03-26T20:57:04.447 INFO:teuthology.orchestra.run.smithi063.stdout: 5 36797 18.53 MB/sec warmup 58 sec latency 20.250 ms 2024-03-26T20:57:04.771 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:57:04 smithi063 ceph-mon[24818]: pgmap v264: 97 pgs: 97 active+clean; 224 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 9.5 KiB/s rd, 8.8 MiB/s wr, 801 op/s 2024-03-26T20:57:05.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:57:04 smithi174 ceph-mon[26934]: pgmap v264: 97 pgs: 97 active+clean; 224 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 9.5 KiB/s rd, 8.8 MiB/s wr, 801 op/s 2024-03-26T20:57:05.448 INFO:teuthology.orchestra.run.smithi063.stdout: 5 37474 18.49 MB/sec warmup 59 sec latency 34.108 ms 2024-03-26T20:57:06.448 INFO:teuthology.orchestra.run.smithi063.stdout: 5 38147 18.45 MB/sec warmup 60 sec latency 75.522 ms 2024-03-26T20:57:06.625 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:57:06 smithi063 ceph-mon[24818]: pgmap v265: 97 pgs: 97 active+clean; 232 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 8.1 MiB/s wr, 901 op/s 2024-03-26T20:57:06.792 INFO:teuthology.orchestra.run.smithi063.stdout:true 2024-03-26T20:57:07.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:57:06 smithi174 ceph-mon[26934]: pgmap v265: 97 pgs: 97 active+clean; 232 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 8.1 MiB/s wr, 901 op/s 2024-03-26T20:57:07.448 INFO:teuthology.orchestra.run.smithi063.stdout: 5 38759 18.55 MB/sec warmup 61 sec latency 55.193 ms 2024-03-26T20:57:07.650 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:57:07 smithi063 ceph-mon[24818]: from='client.14620 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:57:07.651 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:57:07 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi063.lxhujt/trash_purge_schedule"}]: dispatch 2024-03-26T20:57:07.651 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:57:07 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi063.lxhujt/mirror_snapshot_schedule"}]: dispatch 2024-03-26T20:57:07.809 INFO:teuthology.orchestra.run.smithi063.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-26T20:57:07.809 INFO:teuthology.orchestra.run.smithi063.stdout:alertmanager.smithi063 smithi063 *:9093,9094 running (6m) 63s ago 7m 22.9M - ba2b418f427c f01f1c52094d 2024-03-26T20:57:07.809 INFO:teuthology.orchestra.run.smithi063.stdout:crash.smithi063 smithi063 running (7m) 63s ago 7m 6409k - 17.2.0 e1d6a67b021e 706c62f8af89 2024-03-26T20:57:07.809 INFO:teuthology.orchestra.run.smithi063.stdout:crash.smithi174 smithi174 running (6m) 65s ago 6m 6409k - 17.2.0 e1d6a67b021e 2c6bfe6d0ca6 2024-03-26T20:57:07.809 INFO:teuthology.orchestra.run.smithi063.stdout:grafana.smithi063 smithi063 *:3000 running (5m) 63s ago 7m 51.8M - 8.3.5 dad864ee21e9 74e19c64e1c4 2024-03-26T20:57:07.809 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foofs.smithi063.tzaznz smithi063 running (93s) 63s ago 92s 14.9M - 17.2.0 e1d6a67b021e 10a989fc7954 2024-03-26T20:57:07.809 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foofs.smithi174.gitdvk smithi174 running (90s) 65s ago 90s 11.1M - 17.2.0 e1d6a67b021e 4339a5d98d1d 2024-03-26T20:57:07.809 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.smithi063.lxhujt smithi063 *:9283 running (8m) 63s ago 8m 459M - 17.2.0 e1d6a67b021e 903406c7bde7 2024-03-26T20:57:07.810 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.smithi174.sexoin smithi174 *:8443,9283 running (6m) 65s ago 6m 413M - 17.2.0 e1d6a67b021e ccd5e4d16e93 2024-03-26T20:57:07.810 INFO:teuthology.orchestra.run.smithi063.stdout:mon.smithi063 smithi063 running (8m) 63s ago 8m 47.4M 2048M 17.2.0 e1d6a67b021e e6bd58b6c0f9 2024-03-26T20:57:07.810 INFO:teuthology.orchestra.run.smithi063.stdout:mon.smithi174 smithi174 running (6m) 65s ago 6m 37.3M 2048M 17.2.0 e1d6a67b021e 0f81c09f0c0a 2024-03-26T20:57:07.810 INFO:teuthology.orchestra.run.smithi063.stdout:nfs.foo.0.0.smithi063.fgmpwa smithi063 *:2049 running (74s) 63s ago 73s 47.0M - 4.0 e1d6a67b021e 3d6c0dd9ce51 2024-03-26T20:57:07.810 INFO:teuthology.orchestra.run.smithi063.stdout:nfs.foo.1.0.smithi174.pytptt smithi174 *:2049 running (68s) 65s ago 68s 48.7M - 4.0 e1d6a67b021e f220dbb449a3 2024-03-26T20:57:07.810 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.smithi063 smithi063 *:9100 running (7m) 63s ago 7m 15.9M - 1dbe0e931976 17d79ef02892 2024-03-26T20:57:07.810 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.smithi174 smithi174 *:9100 running (6m) 65s ago 6m 20.1M - 1dbe0e931976 6582440a94dc 2024-03-26T20:57:07.810 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0 smithi063 running (6m) 63s ago 6m 42.7M 3216M 17.2.0 e1d6a67b021e 761b1388d8fa 2024-03-26T20:57:07.810 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1 smithi063 running (5m) 63s ago 5m 42.2M 3216M 17.2.0 e1d6a67b021e 108275c17cad 2024-03-26T20:57:07.810 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2 smithi063 running (5m) 63s ago 5m 40.3M 3216M 17.2.0 e1d6a67b021e 107f685ad08d 2024-03-26T20:57:07.810 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3 smithi063 running (4m) 63s ago 4m 42.7M 3216M 17.2.0 e1d6a67b021e ab0f1a855957 2024-03-26T20:57:07.810 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4 smithi174 running (4m) 65s ago 4m 39.1M 3984M 17.2.0 e1d6a67b021e a0e877ecc06d 2024-03-26T20:57:07.810 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5 smithi174 running (3m) 65s ago 3m 41.7M 3984M 17.2.0 e1d6a67b021e 378c2aca7e71 2024-03-26T20:57:07.810 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6 smithi174 running (3m) 65s ago 3m 42.0M 3984M 17.2.0 e1d6a67b021e ffbb5b7ad3b6 2024-03-26T20:57:07.811 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7 smithi174 running (2m) 65s ago 2m 38.9M 3984M 17.2.0 e1d6a67b021e c1d57fcabb3f 2024-03-26T20:57:07.811 INFO:teuthology.orchestra.run.smithi063.stdout:prometheus.smithi063 smithi063 *:9095 running (6m) 63s ago 6m 60.4M - 514e6a882f6e a93dcc31497d 2024-03-26T20:57:08.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:57:07 smithi174 ceph-mon[26934]: from='client.14620 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:57:08.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:57:07 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi063.lxhujt/trash_purge_schedule"}]: dispatch 2024-03-26T20:57:08.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:57:07 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi063.lxhujt/mirror_snapshot_schedule"}]: dispatch 2024-03-26T20:57:08.438 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-03-26T20:57:08.438 INFO:teuthology.orchestra.run.smithi063.stdout: "mon": { 2024-03-26T20:57:08.439 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-26T20:57:08.439 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T20:57:08.439 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr": { 2024-03-26T20:57:08.439 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-26T20:57:08.439 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T20:57:08.439 INFO:teuthology.orchestra.run.smithi063.stdout: "osd": { 2024-03-26T20:57:08.439 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-03-26T20:57:08.439 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T20:57:08.439 INFO:teuthology.orchestra.run.smithi063.stdout: "mds": { 2024-03-26T20:57:08.439 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-26T20:57:08.439 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T20:57:08.439 INFO:teuthology.orchestra.run.smithi063.stdout: "overall": { 2024-03-26T20:57:08.439 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-03-26T20:57:08.439 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-03-26T20:57:08.440 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-03-26T20:57:08.448 INFO:teuthology.orchestra.run.smithi063.stdout: 5 39468 18.61 MB/sec warmup 62 sec latency 28.378 ms 2024-03-26T20:57:08.655 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:57:08 smithi063 ceph-mon[24818]: from='client.24417 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:57:08.655 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:57:08 smithi063 ceph-mon[24818]: pgmap v266: 97 pgs: 97 active+clean; 233 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 9.7 KiB/s rd, 9.2 MiB/s wr, 864 op/s 2024-03-26T20:57:08.655 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:57:08 smithi063 ceph-mon[24818]: from='client.14624 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:57:08.655 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:57:08 smithi063 ceph-mon[24818]: from='client.? 172.21.15.63:0/1491624032' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T20:57:08.972 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-03-26T20:57:08.972 INFO:teuthology.orchestra.run.smithi063.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:42fc52b86b14d4f88c40d75eb7102e0ab3918f55", 2024-03-26T20:57:08.972 INFO:teuthology.orchestra.run.smithi063.stdout: "in_progress": true, 2024-03-26T20:57:08.972 INFO:teuthology.orchestra.run.smithi063.stdout: "services_complete": [], 2024-03-26T20:57:08.972 INFO:teuthology.orchestra.run.smithi063.stdout: "progress": "", 2024-03-26T20:57:08.972 INFO:teuthology.orchestra.run.smithi063.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:42fc52b86b14d4f88c40d75eb7102e0ab3918f55 image" 2024-03-26T20:57:08.972 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-03-26T20:57:09.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:57:08 smithi174 ceph-mon[26934]: from='client.24417 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:57:09.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:57:08 smithi174 ceph-mon[26934]: pgmap v266: 97 pgs: 97 active+clean; 233 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 9.7 KiB/s rd, 9.2 MiB/s wr, 864 op/s 2024-03-26T20:57:09.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:57:08 smithi174 ceph-mon[26934]: from='client.14624 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:57:09.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:57:08 smithi174 ceph-mon[26934]: from='client.? 172.21.15.63:0/1491624032' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T20:57:09.448 INFO:teuthology.orchestra.run.smithi063.stdout: 5 39945 18.49 MB/sec warmup 63 sec latency 66.102 ms 2024-03-26T20:57:09.552 INFO:teuthology.orchestra.run.smithi063.stdout:HEALTH_OK 2024-03-26T20:57:09.703 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:57:09 smithi063 ceph-mon[24818]: from='client.? 172.21.15.63:0/2420090667' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-26T20:57:10.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:57:09 smithi174 ceph-mon[26934]: from='client.? 172.21.15.63:0/2420090667' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-26T20:57:10.448 INFO:teuthology.orchestra.run.smithi063.stdout: 5 40625 18.35 MB/sec warmup 64 sec latency 38.503 ms 2024-03-26T20:57:10.708 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:57:10 smithi063 ceph-mon[24818]: from='client.24429 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:57:10.708 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:57:10 smithi063 ceph-mon[24818]: pgmap v267: 97 pgs: 97 active+clean; 237 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.7 KiB/s rd, 9.1 MiB/s wr, 791 op/s 2024-03-26T20:57:11.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:57:10 smithi174 ceph-mon[26934]: from='client.24429 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:57:11.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:57:10 smithi174 ceph-mon[26934]: pgmap v267: 97 pgs: 97 active+clean; 237 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.7 KiB/s rd, 9.1 MiB/s wr, 791 op/s 2024-03-26T20:57:11.448 INFO:teuthology.orchestra.run.smithi063.stdout: 5 41300 18.37 MB/sec warmup 65 sec latency 24.347 ms 2024-03-26T20:57:12.448 INFO:teuthology.orchestra.run.smithi063.stdout: 5 42073 18.44 MB/sec warmup 66 sec latency 38.045 ms 2024-03-26T20:57:12.980 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:57:12 smithi063 ceph-mon[24818]: pgmap v268: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 9.7 MiB/s wr, 966 op/s 2024-03-26T20:57:12.980 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:57:12 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:57:12.980 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:57:12 smithi063 ceph-mon[24818]: Upgrade: Target is version 19.0.0-2484-g42fc52b8 (unknown) 2024-03-26T20:57:12.980 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:57:12 smithi063 ceph-mon[24818]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d'] 2024-03-26T20:57:12.980 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:57:12 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T20:57:12.980 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:57:12 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:57:12.980 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:57:12 smithi063 ceph-mon[24818]: Upgrade: Need to upgrade myself (mgr.smithi063.lxhujt) 2024-03-26T20:57:13.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:57:12 smithi174 ceph-mon[26934]: pgmap v268: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 9.7 MiB/s wr, 966 op/s 2024-03-26T20:57:13.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:57:12 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:57:13.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:57:12 smithi174 ceph-mon[26934]: Upgrade: Target is version 19.0.0-2484-g42fc52b8 (unknown) 2024-03-26T20:57:13.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:57:12 smithi174 ceph-mon[26934]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d'] 2024-03-26T20:57:13.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:57:12 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T20:57:13.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:57:12 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:57:13.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:57:12 smithi174 ceph-mon[26934]: Upgrade: Need to upgrade myself (mgr.smithi063.lxhujt) 2024-03-26T20:57:13.448 INFO:teuthology.orchestra.run.smithi063.stdout: 5 42616 18.49 MB/sec warmup 67 sec latency 33.505 ms 2024-03-26T20:57:14.007 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:57:13 smithi063 ceph-mon[24818]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d on smithi174 2024-03-26T20:57:14.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:57:13 smithi174 ceph-mon[26934]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d on smithi174 2024-03-26T20:57:14.448 INFO:teuthology.orchestra.run.smithi063.stdout: 5 43210 18.44 MB/sec warmup 68 sec latency 21.888 ms 2024-03-26T20:57:14.959 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:57:14 smithi063 ceph-mon[24818]: pgmap v269: 97 pgs: 97 active+clean; 243 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 10 MiB/s wr, 903 op/s 2024-03-26T20:57:15.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:57:14 smithi174 ceph-mon[26934]: pgmap v269: 97 pgs: 97 active+clean; 243 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 10 MiB/s wr, 903 op/s 2024-03-26T20:57:15.448 INFO:teuthology.orchestra.run.smithi063.stdout: 5 43945 18.39 MB/sec warmup 69 sec latency 19.792 ms 2024-03-26T20:57:16.449 INFO:teuthology.orchestra.run.smithi063.stdout: 5 44677 18.34 MB/sec warmup 70 sec latency 26.232 ms 2024-03-26T20:57:16.973 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:57:16 smithi063 ceph-mon[24818]: pgmap v270: 97 pgs: 97 active+clean; 250 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 8.8 MiB/s wr, 828 op/s 2024-03-26T20:57:17.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:57:16 smithi174 ceph-mon[26934]: pgmap v270: 97 pgs: 97 active+clean; 250 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 8.8 MiB/s wr, 828 op/s 2024-03-26T20:57:17.448 INFO:teuthology.orchestra.run.smithi063.stdout: 5 45618 18.49 MB/sec warmup 71 sec latency 26.963 ms 2024-03-26T20:57:18.448 INFO:teuthology.orchestra.run.smithi063.stdout: 5 46385 18.59 MB/sec warmup 72 sec latency 21.849 ms 2024-03-26T20:57:18.479 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:57:18 smithi063 ceph-mon[24818]: pgmap v271: 97 pgs: 97 active+clean; 259 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 10 MiB/s wr, 898 op/s 2024-03-26T20:57:18.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:57:18 smithi174 ceph-mon[26934]: pgmap v271: 97 pgs: 97 active+clean; 259 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 10 MiB/s wr, 898 op/s 2024-03-26T20:57:19.448 INFO:teuthology.orchestra.run.smithi063.stdout: 5 47208 18.59 MB/sec warmup 73 sec latency 21.582 ms 2024-03-26T20:57:20.448 INFO:teuthology.orchestra.run.smithi063.stdout: 5 48109 18.61 MB/sec warmup 74 sec latency 19.685 ms 2024-03-26T20:57:20.740 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:57:20 smithi063 ceph-mon[24818]: pgmap v272: 97 pgs: 97 active+clean; 265 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 10 MiB/s wr, 950 op/s 2024-03-26T20:57:21.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:57:20 smithi174 ceph-mon[26934]: pgmap v272: 97 pgs: 97 active+clean; 265 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 10 MiB/s wr, 950 op/s 2024-03-26T20:57:21.448 INFO:teuthology.orchestra.run.smithi063.stdout: 5 48887 18.65 MB/sec warmup 75 sec latency 22.806 ms 2024-03-26T20:57:22.449 INFO:teuthology.orchestra.run.smithi063.stdout: 5 49537 18.75 MB/sec warmup 76 sec latency 24.081 ms 2024-03-26T20:57:22.756 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:57:22 smithi063 ceph-mon[24818]: pgmap v273: 97 pgs: 97 active+clean; 269 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 10 MiB/s wr, 1.03k op/s 2024-03-26T20:57:23.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:57:22 smithi174 ceph-mon[26934]: pgmap v273: 97 pgs: 97 active+clean; 269 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 10 MiB/s wr, 1.03k op/s 2024-03-26T20:57:23.449 INFO:teuthology.orchestra.run.smithi063.stdout: 5 50036 18.70 MB/sec warmup 77 sec latency 42.935 ms 2024-03-26T20:57:24.449 INFO:teuthology.orchestra.run.smithi063.stdout: 5 50547 18.60 MB/sec warmup 78 sec latency 22.511 ms 2024-03-26T20:57:24.788 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:57:24 smithi063 ceph-mon[24818]: pgmap v274: 97 pgs: 97 active+clean; 276 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 11 MiB/s wr, 985 op/s 2024-03-26T20:57:25.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:57:24 smithi174 ceph-mon[26934]: pgmap v274: 97 pgs: 97 active+clean; 276 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 11 MiB/s wr, 985 op/s 2024-03-26T20:57:25.449 INFO:teuthology.orchestra.run.smithi063.stdout: 5 51245 18.51 MB/sec warmup 79 sec latency 24.024 ms 2024-03-26T20:57:26.449 INFO:teuthology.orchestra.run.smithi063.stdout: 5 51915 18.50 MB/sec warmup 80 sec latency 18.161 ms 2024-03-26T20:57:26.805 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:57:26 smithi063 ceph-mon[24818]: pgmap v275: 97 pgs: 97 active+clean; 274 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 9.5 MiB/s wr, 911 op/s 2024-03-26T20:57:27.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:57:26 smithi174 ceph-mon[26934]: pgmap v275: 97 pgs: 97 active+clean; 274 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 9.5 MiB/s wr, 911 op/s 2024-03-26T20:57:27.449 INFO:teuthology.orchestra.run.smithi063.stdout: 5 52599 18.52 MB/sec warmup 81 sec latency 59.117 ms 2024-03-26T20:57:28.449 INFO:teuthology.orchestra.run.smithi063.stdout: 5 53196 18.60 MB/sec warmup 82 sec latency 23.524 ms 2024-03-26T20:57:28.577 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:57:28 smithi063 ceph-mon[24818]: pgmap v276: 97 pgs: 97 active+clean; 276 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 10 MiB/s wr, 908 op/s 2024-03-26T20:57:28.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:57:28 smithi174 ceph-mon[26934]: pgmap v276: 97 pgs: 97 active+clean; 276 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 10 MiB/s wr, 908 op/s 2024-03-26T20:57:29.456 INFO:teuthology.orchestra.run.smithi063.stdout: 5 53797 18.55 MB/sec warmup 83 sec latency 27.933 ms 2024-03-26T20:57:30.456 INFO:teuthology.orchestra.run.smithi063.stdout: 5 54284 18.43 MB/sec warmup 84 sec latency 53.212 ms 2024-03-26T20:57:30.842 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:57:30 smithi063 ceph-mon[24818]: pgmap v277: 97 pgs: 97 active+clean; 278 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 9.7 MiB/s wr, 804 op/s 2024-03-26T20:57:31.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:57:30 smithi174 ceph-mon[26934]: pgmap v277: 97 pgs: 97 active+clean; 278 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 9.7 MiB/s wr, 804 op/s 2024-03-26T20:57:31.456 INFO:teuthology.orchestra.run.smithi063.stdout: 5 54845 18.34 MB/sec warmup 85 sec latency 66.944 ms 2024-03-26T20:57:32.456 INFO:teuthology.orchestra.run.smithi063.stdout: 5 55435 18.31 MB/sec warmup 86 sec latency 32.869 ms 2024-03-26T20:57:32.653 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:57:32 smithi063 ceph-mon[24818]: pgmap v278: 97 pgs: 97 active+clean; 287 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 9.3 MiB/s wr, 853 op/s 2024-03-26T20:57:33.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:57:32 smithi174 ceph-mon[26934]: pgmap v278: 97 pgs: 97 active+clean; 287 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 9.3 MiB/s wr, 853 op/s 2024-03-26T20:57:33.456 INFO:teuthology.orchestra.run.smithi063.stdout: 5 56067 18.33 MB/sec warmup 87 sec latency 25.475 ms 2024-03-26T20:57:34.456 INFO:teuthology.orchestra.run.smithi063.stdout: 5 56635 18.38 MB/sec warmup 88 sec latency 36.434 ms 2024-03-26T20:57:34.678 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:57:34 smithi063 ceph-mon[24818]: pgmap v279: 97 pgs: 97 active+clean; 288 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.8 MiB/s wr, 866 op/s 2024-03-26T20:57:35.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:57:34 smithi174 ceph-mon[26934]: pgmap v279: 97 pgs: 97 active+clean; 288 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.8 MiB/s wr, 866 op/s 2024-03-26T20:57:35.456 INFO:teuthology.orchestra.run.smithi063.stdout: 5 57113 18.33 MB/sec warmup 89 sec latency 33.827 ms 2024-03-26T20:57:36.456 INFO:teuthology.orchestra.run.smithi063.stdout: 5 57445 18.18 MB/sec warmup 90 sec latency 33.344 ms 2024-03-26T20:57:36.713 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:57:36 smithi063 ceph-mon[24818]: pgmap v280: 97 pgs: 97 active+clean; 291 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 7.5 MiB/s wr, 749 op/s 2024-03-26T20:57:37.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:57:36 smithi174 ceph-mon[26934]: pgmap v280: 97 pgs: 97 active+clean; 291 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 7.5 MiB/s wr, 749 op/s 2024-03-26T20:57:37.456 INFO:teuthology.orchestra.run.smithi063.stdout: 5 57788 18.06 MB/sec warmup 91 sec latency 44.041 ms 2024-03-26T20:57:38.457 INFO:teuthology.orchestra.run.smithi063.stdout: 5 58284 17.97 MB/sec warmup 92 sec latency 44.251 ms 2024-03-26T20:57:38.497 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:57:38 smithi063 ceph-mon[24818]: pgmap v281: 97 pgs: 97 active+clean; 298 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 9.9 KiB/s rd, 9.1 MiB/s wr, 900 op/s 2024-03-26T20:57:38.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:57:38 smithi174 ceph-mon[26934]: pgmap v281: 97 pgs: 97 active+clean; 298 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 9.9 KiB/s rd, 9.1 MiB/s wr, 900 op/s 2024-03-26T20:57:39.457 INFO:teuthology.orchestra.run.smithi063.stdout: 5 58724 17.86 MB/sec warmup 93 sec latency 41.362 ms 2024-03-26T20:57:39.993 INFO:teuthology.orchestra.run.smithi063.stdout:true 2024-03-26T20:57:40.457 INFO:teuthology.orchestra.run.smithi063.stdout: 5 59128 17.80 MB/sec warmup 94 sec latency 33.680 ms 2024-03-26T20:57:40.786 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:57:40 smithi063 ceph-mon[24818]: pgmap v282: 97 pgs: 97 active+clean; 302 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 9.9 KiB/s rd, 8.7 MiB/s wr, 863 op/s 2024-03-26T20:57:40.790 INFO:teuthology.orchestra.run.smithi063.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-26T20:57:40.790 INFO:teuthology.orchestra.run.smithi063.stdout:alertmanager.smithi063 smithi063 *:9093,9094 running (6m) 96s ago 8m 22.9M - ba2b418f427c f01f1c52094d 2024-03-26T20:57:40.790 INFO:teuthology.orchestra.run.smithi063.stdout:crash.smithi063 smithi063 running (7m) 96s ago 7m 6409k - 17.2.0 e1d6a67b021e 706c62f8af89 2024-03-26T20:57:40.790 INFO:teuthology.orchestra.run.smithi063.stdout:crash.smithi174 smithi174 running (7m) 98s ago 7m 6409k - 17.2.0 e1d6a67b021e 2c6bfe6d0ca6 2024-03-26T20:57:40.790 INFO:teuthology.orchestra.run.smithi063.stdout:grafana.smithi063 smithi063 *:3000 running (6m) 96s ago 7m 51.8M - 8.3.5 dad864ee21e9 74e19c64e1c4 2024-03-26T20:57:40.791 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foofs.smithi063.tzaznz smithi063 running (2m) 96s ago 2m 14.9M - 17.2.0 e1d6a67b021e 10a989fc7954 2024-03-26T20:57:40.791 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foofs.smithi174.gitdvk smithi174 running (2m) 98s ago 2m 11.1M - 17.2.0 e1d6a67b021e 4339a5d98d1d 2024-03-26T20:57:40.791 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.smithi063.lxhujt smithi063 *:9283 running (9m) 96s ago 9m 459M - 17.2.0 e1d6a67b021e 903406c7bde7 2024-03-26T20:57:40.791 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.smithi174.sexoin smithi174 *:8443,9283 running (7m) 98s ago 7m 413M - 17.2.0 e1d6a67b021e ccd5e4d16e93 2024-03-26T20:57:40.791 INFO:teuthology.orchestra.run.smithi063.stdout:mon.smithi063 smithi063 running (9m) 96s ago 9m 47.4M 2048M 17.2.0 e1d6a67b021e e6bd58b6c0f9 2024-03-26T20:57:40.791 INFO:teuthology.orchestra.run.smithi063.stdout:mon.smithi174 smithi174 running (7m) 98s ago 7m 37.3M 2048M 17.2.0 e1d6a67b021e 0f81c09f0c0a 2024-03-26T20:57:40.791 INFO:teuthology.orchestra.run.smithi063.stdout:nfs.foo.0.0.smithi063.fgmpwa smithi063 *:2049 running (107s) 96s ago 106s 47.0M - 4.0 e1d6a67b021e 3d6c0dd9ce51 2024-03-26T20:57:40.791 INFO:teuthology.orchestra.run.smithi063.stdout:nfs.foo.1.0.smithi174.pytptt smithi174 *:2049 running (101s) 98s ago 101s 48.7M - 4.0 e1d6a67b021e f220dbb449a3 2024-03-26T20:57:40.791 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.smithi063 smithi063 *:9100 running (7m) 96s ago 7m 15.9M - 1dbe0e931976 17d79ef02892 2024-03-26T20:57:40.791 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.smithi174 smithi174 *:9100 running (7m) 98s ago 7m 20.1M - 1dbe0e931976 6582440a94dc 2024-03-26T20:57:40.791 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0 smithi063 running (6m) 96s ago 6m 42.7M 3216M 17.2.0 e1d6a67b021e 761b1388d8fa 2024-03-26T20:57:40.791 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1 smithi063 running (6m) 96s ago 6m 42.2M 3216M 17.2.0 e1d6a67b021e 108275c17cad 2024-03-26T20:57:40.791 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2 smithi063 running (5m) 96s ago 5m 40.3M 3216M 17.2.0 e1d6a67b021e 107f685ad08d 2024-03-26T20:57:40.791 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3 smithi063 running (5m) 96s ago 5m 42.7M 3216M 17.2.0 e1d6a67b021e ab0f1a855957 2024-03-26T20:57:40.791 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4 smithi174 running (4m) 98s ago 4m 39.1M 3984M 17.2.0 e1d6a67b021e a0e877ecc06d 2024-03-26T20:57:40.792 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5 smithi174 running (4m) 98s ago 4m 41.7M 3984M 17.2.0 e1d6a67b021e 378c2aca7e71 2024-03-26T20:57:40.792 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6 smithi174 running (3m) 98s ago 3m 42.0M 3984M 17.2.0 e1d6a67b021e ffbb5b7ad3b6 2024-03-26T20:57:40.792 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7 smithi174 running (3m) 98s ago 3m 38.9M 3984M 17.2.0 e1d6a67b021e c1d57fcabb3f 2024-03-26T20:57:40.792 INFO:teuthology.orchestra.run.smithi063.stdout:prometheus.smithi063 smithi063 *:9095 running (6m) 96s ago 6m 60.4M - 514e6a882f6e a93dcc31497d 2024-03-26T20:57:41.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:57:40 smithi174 ceph-mon[26934]: pgmap v282: 97 pgs: 97 active+clean; 302 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 9.9 KiB/s rd, 8.7 MiB/s wr, 863 op/s 2024-03-26T20:57:41.263 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-03-26T20:57:41.263 INFO:teuthology.orchestra.run.smithi063.stdout: "mon": { 2024-03-26T20:57:41.263 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-26T20:57:41.263 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T20:57:41.263 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr": { 2024-03-26T20:57:41.263 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-26T20:57:41.263 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T20:57:41.264 INFO:teuthology.orchestra.run.smithi063.stdout: "osd": { 2024-03-26T20:57:41.264 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-03-26T20:57:41.264 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T20:57:41.264 INFO:teuthology.orchestra.run.smithi063.stdout: "mds": { 2024-03-26T20:57:41.264 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-26T20:57:41.264 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T20:57:41.264 INFO:teuthology.orchestra.run.smithi063.stdout: "overall": { 2024-03-26T20:57:41.264 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-03-26T20:57:41.264 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-03-26T20:57:41.264 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-03-26T20:57:41.457 INFO:teuthology.orchestra.run.smithi063.stdout: 5 59588 17.79 MB/sec warmup 95 sec latency 36.839 ms 2024-03-26T20:57:41.693 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-03-26T20:57:41.694 INFO:teuthology.orchestra.run.smithi063.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d", 2024-03-26T20:57:41.694 INFO:teuthology.orchestra.run.smithi063.stdout: "in_progress": true, 2024-03-26T20:57:41.694 INFO:teuthology.orchestra.run.smithi063.stdout: "services_complete": [], 2024-03-26T20:57:41.694 INFO:teuthology.orchestra.run.smithi063.stdout: "progress": "0/23 daemons upgraded", 2024-03-26T20:57:41.694 INFO:teuthology.orchestra.run.smithi063.stdout: "message": "Pulling quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d image on host smithi174" 2024-03-26T20:57:41.694 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-03-26T20:57:41.798 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:57:41 smithi063 ceph-mon[24818]: from='client.14638 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:57:41.799 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:57:41 smithi063 ceph-mon[24818]: from='client.14642 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:57:41.799 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:57:41 smithi063 ceph-mon[24818]: from='client.14646 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:57:41.799 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:57:41 smithi063 ceph-mon[24818]: from='client.? 172.21.15.63:0/778457607' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T20:57:42.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:57:41 smithi174 ceph-mon[26934]: from='client.14638 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:57:42.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:57:41 smithi174 ceph-mon[26934]: from='client.14642 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:57:42.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:57:41 smithi174 ceph-mon[26934]: from='client.14646 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:57:42.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:57:41 smithi174 ceph-mon[26934]: from='client.? 172.21.15.63:0/778457607' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T20:57:42.243 INFO:teuthology.orchestra.run.smithi063.stdout:HEALTH_OK 2024-03-26T20:57:42.457 INFO:teuthology.orchestra.run.smithi063.stdout: 5 60127 17.82 MB/sec warmup 96 sec latency 38.157 ms 2024-03-26T20:57:42.812 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:57:42 smithi063 ceph-mon[24818]: pgmap v283: 97 pgs: 97 active+clean; 299 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 7.7 MiB/s wr, 769 op/s 2024-03-26T20:57:42.812 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:57:42 smithi063 ceph-mon[24818]: from='client.14654 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:57:42.812 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:57:42 smithi063 ceph-mon[24818]: from='client.? 172.21.15.63:0/4212521311' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-26T20:57:43.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:57:42 smithi174 ceph-mon[26934]: pgmap v283: 97 pgs: 97 active+clean; 299 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 7.7 MiB/s wr, 769 op/s 2024-03-26T20:57:43.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:57:42 smithi174 ceph-mon[26934]: from='client.14654 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:57:43.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:57:42 smithi174 ceph-mon[26934]: from='client.? 172.21.15.63:0/4212521311' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-26T20:57:43.457 INFO:teuthology.orchestra.run.smithi063.stdout: 5 60538 17.74 MB/sec warmup 97 sec latency 49.970 ms 2024-03-26T20:57:44.457 INFO:teuthology.orchestra.run.smithi063.stdout: 5 60921 17.67 MB/sec warmup 98 sec latency 50.004 ms 2024-03-26T20:57:44.825 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:57:44 smithi063 ceph-mon[24818]: pgmap v284: 97 pgs: 97 active+clean; 307 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 9.9 KiB/s rd, 7.8 MiB/s wr, 726 op/s 2024-03-26T20:57:44.971 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:57:44 smithi174 ceph-mon[26934]: pgmap v284: 97 pgs: 97 active+clean; 307 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 9.9 KiB/s rd, 7.8 MiB/s wr, 726 op/s 2024-03-26T20:57:45.457 INFO:teuthology.orchestra.run.smithi063.stdout: 5 61257 17.56 MB/sec warmup 99 sec latency 52.518 ms 2024-03-26T20:57:46.457 INFO:teuthology.orchestra.run.smithi063.stdout: 5 61708 17.47 MB/sec warmup 100 sec latency 61.791 ms 2024-03-26T20:57:46.870 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:57:46 smithi063 ceph-mon[24818]: pgmap v285: 97 pgs: 97 active+clean; 308 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 5.9 MiB/s wr, 572 op/s 2024-03-26T20:57:47.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:57:46 smithi174 ceph-mon[26934]: pgmap v285: 97 pgs: 97 active+clean; 308 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 5.9 MiB/s wr, 572 op/s 2024-03-26T20:57:47.457 INFO:teuthology.orchestra.run.smithi063.stdout: 5 62066 17.35 MB/sec warmup 101 sec latency 99.456 ms 2024-03-26T20:57:48.428 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:57:48 smithi063 ceph-mon[24818]: pgmap v286: 97 pgs: 97 active+clean; 313 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 7.9 KiB/s rd, 6.9 MiB/s wr, 625 op/s 2024-03-26T20:57:48.457 INFO:teuthology.orchestra.run.smithi063.stdout: 5 62485 17.32 MB/sec warmup 102 sec latency 50.017 ms 2024-03-26T20:57:48.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:57:48 smithi174 ceph-mon[26934]: pgmap v286: 97 pgs: 97 active+clean; 313 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 7.9 KiB/s rd, 6.9 MiB/s wr, 625 op/s 2024-03-26T20:57:49.457 INFO:teuthology.orchestra.run.smithi063.stdout: 5 62912 17.23 MB/sec warmup 103 sec latency 45.173 ms 2024-03-26T20:57:50.457 INFO:teuthology.orchestra.run.smithi063.stdout: 5 63363 17.22 MB/sec warmup 104 sec latency 60.107 ms 2024-03-26T20:57:50.741 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:57:50 smithi063 ceph-mon[24818]: pgmap v287: 97 pgs: 97 active+clean; 314 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 7.9 KiB/s rd, 6.4 MiB/s wr, 641 op/s 2024-03-26T20:57:51.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:57:50 smithi174 ceph-mon[26934]: pgmap v287: 97 pgs: 97 active+clean; 314 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 7.9 KiB/s rd, 6.4 MiB/s wr, 641 op/s 2024-03-26T20:57:51.457 INFO:teuthology.orchestra.run.smithi063.stdout: 5 63788 17.24 MB/sec warmup 105 sec latency 53.451 ms 2024-03-26T20:57:52.457 INFO:teuthology.orchestra.run.smithi063.stdout: 5 64297 17.20 MB/sec warmup 106 sec latency 34.175 ms 2024-03-26T20:57:52.518 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:57:52 smithi063 ceph-mon[24818]: Upgrade: Updating mgr.smithi174.sexoin 2024-03-26T20:57:52.518 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:57:52 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:57:52.518 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:57:52 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi174.sexoin", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-26T20:57:52.518 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:57:52 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-26T20:57:52.518 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:57:52 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:57:52.518 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:57:52 smithi063 ceph-mon[24818]: Deploying daemon mgr.smithi174.sexoin on smithi174 2024-03-26T20:57:52.518 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:57:52 smithi063 ceph-mon[24818]: pgmap v288: 97 pgs: 97 active+clean; 319 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 6.2 MiB/s wr, 620 op/s 2024-03-26T20:57:52.758 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:57:52 smithi174 ceph-mon[26934]: Upgrade: Updating mgr.smithi174.sexoin 2024-03-26T20:57:52.759 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:57:52 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:57:52.759 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:57:52 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi174.sexoin", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-26T20:57:52.759 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:57:52 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-26T20:57:52.759 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:57:52 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:57:52.759 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:57:52 smithi174 ceph-mon[26934]: Deploying daemon mgr.smithi174.sexoin on smithi174 2024-03-26T20:57:52.759 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:57:52 smithi174 ceph-mon[26934]: pgmap v288: 97 pgs: 97 active+clean; 319 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 6.2 MiB/s wr, 620 op/s 2024-03-26T20:57:53.457 INFO:teuthology.orchestra.run.smithi063.stdout: 5 64676 17.12 MB/sec warmup 107 sec latency 32.211 ms 2024-03-26T20:57:54.457 INFO:teuthology.orchestra.run.smithi063.stdout: 5 65206 17.06 MB/sec warmup 108 sec latency 31.158 ms 2024-03-26T20:57:54.821 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:57:54 smithi063 ceph-mon[24818]: pgmap v289: 97 pgs: 97 active+clean; 327 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 7.0 KiB/s rd, 7.3 MiB/s wr, 710 op/s 2024-03-26T20:57:54.892 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:57:54 smithi174 ceph-mon[26934]: pgmap v289: 97 pgs: 97 active+clean; 327 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 7.0 KiB/s rd, 7.3 MiB/s wr, 710 op/s 2024-03-26T20:57:55.457 INFO:teuthology.orchestra.run.smithi063.stdout: 5 65608 16.96 MB/sec warmup 109 sec latency 148.494 ms 2024-03-26T20:57:56.093 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:57:55 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:57:56.093 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:57:55 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T20:57:56.093 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:57:55 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:57:56.093 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:57:55 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T20:57:56.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:57:55 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:57:56.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:57:55 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T20:57:56.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:57:55 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:57:56.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:57:55 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T20:57:56.457 INFO:teuthology.orchestra.run.smithi063.stdout: 5 66082 16.93 MB/sec warmup 110 sec latency 38.985 ms 2024-03-26T20:57:56.982 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:57:56 smithi063 ceph-mon[24818]: pgmap v290: 97 pgs: 97 active+clean; 328 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 5.6 MiB/s wr, 556 op/s 2024-03-26T20:57:57.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:57:56 smithi174 ceph-mon[26934]: pgmap v290: 97 pgs: 97 active+clean; 328 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 5.6 MiB/s wr, 556 op/s 2024-03-26T20:57:57.457 INFO:teuthology.orchestra.run.smithi063.stdout: 5 66747 16.96 MB/sec warmup 111 sec latency 20.951 ms 2024-03-26T20:57:58.416 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:57:58 smithi063 ceph-mon[24818]: pgmap v291: 97 pgs: 97 active+clean; 328 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 8.0 KiB/s rd, 6.9 MiB/s wr, 732 op/s 2024-03-26T20:57:58.457 INFO:teuthology.orchestra.run.smithi063.stdout: 5 67285 16.99 MB/sec warmup 112 sec latency 42.843 ms 2024-03-26T20:57:58.736 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:57:58 smithi174 ceph-mon[26934]: pgmap v291: 97 pgs: 97 active+clean; 328 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 8.0 KiB/s rd, 6.9 MiB/s wr, 732 op/s 2024-03-26T20:57:59.457 INFO:teuthology.orchestra.run.smithi063.stdout: 5 67664 16.92 MB/sec warmup 113 sec latency 31.318 ms 2024-03-26T20:58:00.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:57:59 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:58:00.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:57:59 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:58:00.203 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:57:59 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:58:00.204 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:57:59 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:58:00.457 INFO:teuthology.orchestra.run.smithi063.stdout: 5 68021 16.87 MB/sec warmup 114 sec latency 36.778 ms 2024-03-26T20:58:01.216 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:00 smithi063 ceph-mon[24818]: pgmap v292: 97 pgs: 97 active+clean; 327 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 8.0 KiB/s rd, 6.4 MiB/s wr, 715 op/s 2024-03-26T20:58:01.260 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:00 smithi174 ceph-mon[26934]: pgmap v292: 97 pgs: 97 active+clean; 327 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 8.0 KiB/s rd, 6.4 MiB/s wr, 715 op/s 2024-03-26T20:58:01.457 INFO:teuthology.orchestra.run.smithi063.stdout: 5 68313 16.77 MB/sec warmup 115 sec latency 72.054 ms 2024-03-26T20:58:02.227 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:01 smithi063 ceph-mon[24818]: pgmap v293: 97 pgs: 97 active+clean; 327 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 6.5 MiB/s wr, 724 op/s 2024-03-26T20:58:02.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:02 smithi174 ceph-mon[26934]: pgmap v293: 97 pgs: 97 active+clean; 327 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 6.5 MiB/s wr, 724 op/s 2024-03-26T20:58:02.457 INFO:teuthology.orchestra.run.smithi063.stdout: 5 68686 16.68 MB/sec warmup 116 sec latency 72.181 ms 2024-03-26T20:58:03.457 INFO:teuthology.orchestra.run.smithi063.stdout: 5 69063 16.59 MB/sec warmup 117 sec latency 38.753 ms 2024-03-26T20:58:04.457 INFO:teuthology.orchestra.run.smithi063.stdout: 5 69489 16.56 MB/sec warmup 118 sec latency 49.878 ms 2024-03-26T20:58:04.726 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:04 smithi174 ceph-mon[26934]: pgmap v294: 97 pgs: 97 active+clean; 321 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.6 KiB/s rd, 6.9 MiB/s wr, 688 op/s 2024-03-26T20:58:04.726 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:04 smithi174 ceph-mon[26934]: from='mgr.? 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi174.sexoin/crt"}]: dispatch 2024-03-26T20:58:04.726 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:04 smithi174 ceph-mon[26934]: from='mgr.? 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-26T20:58:04.727 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:04 smithi174 ceph-mon[26934]: Standby manager daemon smithi174.sexoin restarted 2024-03-26T20:58:04.727 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:04 smithi174 ceph-mon[26934]: Standby manager daemon smithi174.sexoin started 2024-03-26T20:58:04.727 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:04 smithi174 ceph-mon[26934]: from='mgr.? 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi174.sexoin/key"}]: dispatch 2024-03-26T20:58:04.727 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:04 smithi174 ceph-mon[26934]: from='mgr.? 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-26T20:58:04.727 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:04 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:58:04.727 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:04 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:58:04.727 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:04 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:58:04.727 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:04 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T20:58:04.727 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:04 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mgr fail", "who": "smithi063.lxhujt"}]: dispatch 2024-03-26T20:58:04.727 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:04 smithi174 ceph-mon[26934]: osdmap e69: 8 total, 8 up, 8 in 2024-03-26T20:58:04.765 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:04 smithi063 ceph-mon[24818]: pgmap v294: 97 pgs: 97 active+clean; 321 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.6 KiB/s rd, 6.9 MiB/s wr, 688 op/s 2024-03-26T20:58:04.765 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:04 smithi063 ceph-mon[24818]: from='mgr.? 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi174.sexoin/crt"}]: dispatch 2024-03-26T20:58:04.765 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:04 smithi063 ceph-mon[24818]: from='mgr.? 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-26T20:58:04.765 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:04 smithi063 ceph-mon[24818]: Standby manager daemon smithi174.sexoin restarted 2024-03-26T20:58:04.765 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:04 smithi063 ceph-mon[24818]: Standby manager daemon smithi174.sexoin started 2024-03-26T20:58:04.765 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:04 smithi063 ceph-mon[24818]: from='mgr.? 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi174.sexoin/key"}]: dispatch 2024-03-26T20:58:04.765 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:04 smithi063 ceph-mon[24818]: from='mgr.? 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-26T20:58:04.765 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:04 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:58:04.765 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:04 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:58:04.765 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:04 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' 2024-03-26T20:58:04.765 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:04 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T20:58:04.765 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:04 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mgr fail", "who": "smithi063.lxhujt"}]: dispatch 2024-03-26T20:58:04.766 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:04 smithi063 ceph-mon[24818]: osdmap e69: 8 total, 8 up, 8 in 2024-03-26T20:58:05.457 INFO:teuthology.orchestra.run.smithi063.stdout: 5 70034 16.55 MB/sec warmup 119 sec latency 26.758 ms 2024-03-26T20:58:05.769 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:05 smithi063 ceph-mon[24818]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "mgr fail", "who": "smithi063.lxhujt"}]': finished 2024-03-26T20:58:05.769 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:05 smithi063 ceph-mon[24818]: mgrmap e22: smithi174.sexoin(active, starting, since 0.143903s) 2024-03-26T20:58:05.769 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:05 smithi063 ceph-mon[24818]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "mon metadata", "id": "smithi063"}]: dispatch 2024-03-26T20:58:05.769 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:05 smithi063 ceph-mon[24818]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "mon metadata", "id": "smithi174"}]: dispatch 2024-03-26T20:58:05.769 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:05 smithi063 ceph-mon[24818]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi063.tzaznz"}]: dispatch 2024-03-26T20:58:05.770 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:05 smithi063 ceph-mon[24818]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi174.gitdvk"}]: dispatch 2024-03-26T20:58:05.770 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:05 smithi063 ceph-mon[24818]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "mgr metadata", "who": "smithi174.sexoin", "id": "smithi174.sexoin"}]: dispatch 2024-03-26T20:58:05.770 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:05 smithi063 ceph-mon[24818]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-26T20:58:05.770 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:05 smithi063 ceph-mon[24818]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-26T20:58:05.770 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:05 smithi063 ceph-mon[24818]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-26T20:58:05.770 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:05 smithi063 ceph-mon[24818]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-26T20:58:05.770 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:05 smithi063 ceph-mon[24818]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-26T20:58:05.770 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:05 smithi063 ceph-mon[24818]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-26T20:58:05.770 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:05 smithi063 ceph-mon[24818]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-26T20:58:05.770 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:05 smithi063 ceph-mon[24818]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-26T20:58:05.770 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:05 smithi063 ceph-mon[24818]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-26T20:58:05.770 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:05 smithi063 ceph-mon[24818]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-26T20:58:05.770 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:05 smithi063 ceph-mon[24818]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-26T20:58:05.771 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:05 smithi063 ceph-mon[24818]: Manager daemon smithi174.sexoin is now available 2024-03-26T20:58:05.771 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:05 smithi063 ceph-mon[24818]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:58:05.771 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:05 smithi063 ceph-mon[24818]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T20:58:05.771 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:05 smithi063 ceph-mon[24818]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T20:58:05.771 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:05 smithi063 ceph-mon[24818]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi174.sexoin/mirror_snapshot_schedule"}]: dispatch 2024-03-26T20:58:05.771 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:05 smithi063 ceph-mon[24818]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi174.sexoin/mirror_snapshot_schedule"}]: dispatch 2024-03-26T20:58:05.771 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:05 smithi063 ceph-mon[24818]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi174.sexoin/trash_purge_schedule"}]: dispatch 2024-03-26T20:58:05.771 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:05 smithi063 ceph-mon[24818]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi174.sexoin/trash_purge_schedule"}]: dispatch 2024-03-26T20:58:05.771 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:05 smithi063 ceph-mon[24818]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:58:05.771 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:05 smithi063 ceph-mon[24818]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:58:05.928 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:05 smithi174 ceph-mon[26934]: from='mgr.14204 172.21.15.63:0/1800732415' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "mgr fail", "who": "smithi063.lxhujt"}]': finished 2024-03-26T20:58:05.929 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:05 smithi174 ceph-mon[26934]: mgrmap e22: smithi174.sexoin(active, starting, since 0.143903s) 2024-03-26T20:58:05.929 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:05 smithi174 ceph-mon[26934]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "mon metadata", "id": "smithi063"}]: dispatch 2024-03-26T20:58:05.929 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:05 smithi174 ceph-mon[26934]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "mon metadata", "id": "smithi174"}]: dispatch 2024-03-26T20:58:05.929 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:05 smithi174 ceph-mon[26934]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi063.tzaznz"}]: dispatch 2024-03-26T20:58:05.929 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:05 smithi174 ceph-mon[26934]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi174.gitdvk"}]: dispatch 2024-03-26T20:58:05.929 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:05 smithi174 ceph-mon[26934]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "mgr metadata", "who": "smithi174.sexoin", "id": "smithi174.sexoin"}]: dispatch 2024-03-26T20:58:05.929 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:05 smithi174 ceph-mon[26934]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-26T20:58:05.929 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:05 smithi174 ceph-mon[26934]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-26T20:58:05.929 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:05 smithi174 ceph-mon[26934]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-26T20:58:05.929 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:05 smithi174 ceph-mon[26934]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-26T20:58:05.929 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:05 smithi174 ceph-mon[26934]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-26T20:58:05.929 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:05 smithi174 ceph-mon[26934]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-26T20:58:05.929 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:05 smithi174 ceph-mon[26934]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-26T20:58:05.930 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:05 smithi174 ceph-mon[26934]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-26T20:58:05.930 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:05 smithi174 ceph-mon[26934]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-26T20:58:05.930 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:05 smithi174 ceph-mon[26934]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-26T20:58:05.930 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:05 smithi174 ceph-mon[26934]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-26T20:58:05.930 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:05 smithi174 ceph-mon[26934]: Manager daemon smithi174.sexoin is now available 2024-03-26T20:58:05.930 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:05 smithi174 ceph-mon[26934]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:58:05.930 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:05 smithi174 ceph-mon[26934]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T20:58:05.930 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:05 smithi174 ceph-mon[26934]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T20:58:05.930 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:05 smithi174 ceph-mon[26934]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi174.sexoin/mirror_snapshot_schedule"}]: dispatch 2024-03-26T20:58:05.930 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:05 smithi174 ceph-mon[26934]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi174.sexoin/mirror_snapshot_schedule"}]: dispatch 2024-03-26T20:58:05.930 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:05 smithi174 ceph-mon[26934]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi174.sexoin/trash_purge_schedule"}]: dispatch 2024-03-26T20:58:05.930 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:05 smithi174 ceph-mon[26934]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi174.sexoin/trash_purge_schedule"}]: dispatch 2024-03-26T20:58:05.930 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:05 smithi174 ceph-mon[26934]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:58:05.930 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:05 smithi174 ceph-mon[26934]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:58:06.779 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:06 smithi063 ceph-mon[24818]: Deploying cephadm binary to smithi174 2024-03-26T20:58:06.780 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:06 smithi063 ceph-mon[24818]: Deploying cephadm binary to smithi063 2024-03-26T20:58:06.780 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:06 smithi063 ceph-mon[24818]: mgrmap e23: smithi174.sexoin(active, since 1.16455s) 2024-03-26T20:58:06.780 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:06 smithi063 ceph-mon[24818]: pgmap v3: 97 pgs: 97 active+clean; 326 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-03-26T20:58:07.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:06 smithi174 ceph-mon[26934]: Deploying cephadm binary to smithi174 2024-03-26T20:58:07.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:06 smithi174 ceph-mon[26934]: Deploying cephadm binary to smithi063 2024-03-26T20:58:07.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:06 smithi174 ceph-mon[26934]: mgrmap e23: smithi174.sexoin(active, since 1.16455s) 2024-03-26T20:58:07.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:06 smithi174 ceph-mon[26934]: pgmap v3: 97 pgs: 97 active+clean; 326 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-03-26T20:58:07.457 INFO:teuthology.orchestra.run.smithi063.stdout: 5 71054 20.48 MB/sec execute 1 sec latency 48.121 ms 2024-03-26T20:58:07.794 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:07 smithi063 ceph-mon[24818]: [26/Mar/2024:20:58:06] ENGINE Bus STARTING 2024-03-26T20:58:07.795 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:07 smithi063 ceph-mon[24818]: [26/Mar/2024:20:58:06] ENGINE Serving on http://172.21.15.174:8765 2024-03-26T20:58:07.795 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:07 smithi063 ceph-mon[24818]: [26/Mar/2024:20:58:06] ENGINE Serving on https://172.21.15.174:7150 2024-03-26T20:58:07.795 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:07 smithi063 ceph-mon[24818]: [26/Mar/2024:20:58:06] ENGINE Bus STARTED 2024-03-26T20:58:07.795 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:07 smithi063 ceph-mon[24818]: pgmap v4: 97 pgs: 97 active+clean; 326 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-03-26T20:58:07.795 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:07 smithi063 ceph-mon[24818]: mgrmap e24: smithi174.sexoin(active, since 2s) 2024-03-26T20:58:08.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:07 smithi174 ceph-mon[26934]: [26/Mar/2024:20:58:06] ENGINE Bus STARTING 2024-03-26T20:58:08.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:07 smithi174 ceph-mon[26934]: [26/Mar/2024:20:58:06] ENGINE Serving on http://172.21.15.174:8765 2024-03-26T20:58:08.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:07 smithi174 ceph-mon[26934]: [26/Mar/2024:20:58:06] ENGINE Serving on https://172.21.15.174:7150 2024-03-26T20:58:08.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:07 smithi174 ceph-mon[26934]: [26/Mar/2024:20:58:06] ENGINE Bus STARTED 2024-03-26T20:58:08.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:07 smithi174 ceph-mon[26934]: pgmap v4: 97 pgs: 97 active+clean; 326 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-03-26T20:58:08.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:07 smithi174 ceph-mon[26934]: mgrmap e24: smithi174.sexoin(active, since 2s) 2024-03-26T20:58:08.457 INFO:teuthology.orchestra.run.smithi063.stdout: 5 71560 17.04 MB/sec execute 2 sec latency 34.413 ms 2024-03-26T20:58:09.457 INFO:teuthology.orchestra.run.smithi063.stdout: 5 71952 14.15 MB/sec execute 3 sec latency 30.275 ms 2024-03-26T20:58:09.825 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:09 smithi063 ceph-mon[24818]: pgmap v5: 97 pgs: 97 active+clean; 326 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-03-26T20:58:09.826 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:09 smithi063 ceph-mon[24818]: mgrmap e25: smithi174.sexoin(active, since 4s) 2024-03-26T20:58:10.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:09 smithi174 ceph-mon[26934]: pgmap v5: 97 pgs: 97 active+clean; 326 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-03-26T20:58:10.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:09 smithi174 ceph-mon[26934]: mgrmap e25: smithi174.sexoin(active, since 4s) 2024-03-26T20:58:10.457 INFO:teuthology.orchestra.run.smithi063.stdout: 5 72497 13.05 MB/sec execute 4 sec latency 44.678 ms 2024-03-26T20:58:11.458 INFO:teuthology.orchestra.run.smithi063.stdout: 5 73089 13.41 MB/sec execute 5 sec latency 32.408 ms 2024-03-26T20:58:11.888 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:11 smithi063 ceph-mon[24818]: pgmap v6: 97 pgs: 97 active+clean; 326 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-03-26T20:58:12.050 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:11 smithi174 ceph-mon[26934]: pgmap v6: 97 pgs: 97 active+clean; 326 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-03-26T20:58:12.458 INFO:teuthology.orchestra.run.smithi063.stdout: 5 73629 14.14 MB/sec execute 6 sec latency 108.406 ms 2024-03-26T20:58:12.856 INFO:teuthology.orchestra.run.smithi063.stdout:true 2024-03-26T20:58:13.432 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:13 smithi063 ceph-mon[24818]: pgmap v7: 97 pgs: 97 active+clean; 329 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 37 KiB/s rd, 5.5 MiB/s wr, 540 op/s 2024-03-26T20:58:13.457 INFO:teuthology.orchestra.run.smithi063.stdout: 5 74098 13.71 MB/sec execute 7 sec latency 22.239 ms 2024-03-26T20:58:13.710 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:13 smithi174 ceph-mon[26934]: pgmap v7: 97 pgs: 97 active+clean; 329 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 37 KiB/s rd, 5.5 MiB/s wr, 540 op/s 2024-03-26T20:58:13.874 INFO:teuthology.orchestra.run.smithi063.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-26T20:58:13.874 INFO:teuthology.orchestra.run.smithi063.stdout:alertmanager.smithi063 smithi063 *:9093,9094 running (7m) 2m ago 8m 22.9M - ba2b418f427c f01f1c52094d 2024-03-26T20:58:13.874 INFO:teuthology.orchestra.run.smithi063.stdout:crash.smithi063 smithi063 running (8m) 2m ago 8m 6409k - 17.2.0 e1d6a67b021e 706c62f8af89 2024-03-26T20:58:13.874 INFO:teuthology.orchestra.run.smithi063.stdout:crash.smithi174 smithi174 running (7m) 14s ago 7m 6409k - 17.2.0 e1d6a67b021e 2c6bfe6d0ca6 2024-03-26T20:58:13.874 INFO:teuthology.orchestra.run.smithi063.stdout:grafana.smithi063 smithi063 *:3000 running (6m) 2m ago 8m 51.8M - 8.3.5 dad864ee21e9 74e19c64e1c4 2024-03-26T20:58:13.875 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foofs.smithi063.tzaznz smithi063 running (2m) 2m ago 2m 14.9M - 17.2.0 e1d6a67b021e 10a989fc7954 2024-03-26T20:58:13.875 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foofs.smithi174.gitdvk smithi174 running (2m) 14s ago 2m 11.5M - 17.2.0 e1d6a67b021e 4339a5d98d1d 2024-03-26T20:58:13.875 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.smithi063.lxhujt smithi063 *:9283 running (9m) 2m ago 9m 459M - 17.2.0 e1d6a67b021e 903406c7bde7 2024-03-26T20:58:13.875 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.smithi174.sexoin smithi174 *:8443,9283 running (19s) 14s ago 7m 46.7M - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 c7ed959f9018 2024-03-26T20:58:13.875 INFO:teuthology.orchestra.run.smithi063.stdout:mon.smithi063 smithi063 running (9m) 2m ago 9m 47.4M 2048M 17.2.0 e1d6a67b021e e6bd58b6c0f9 2024-03-26T20:58:13.875 INFO:teuthology.orchestra.run.smithi063.stdout:mon.smithi174 smithi174 running (7m) 14s ago 7m 37.8M 2048M 17.2.0 e1d6a67b021e 0f81c09f0c0a 2024-03-26T20:58:13.875 INFO:teuthology.orchestra.run.smithi063.stdout:nfs.foo.0.0.smithi063.fgmpwa smithi063 *:2049 running (2m) 2m ago 2m 47.0M - 4.0 e1d6a67b021e 3d6c0dd9ce51 2024-03-26T20:58:13.875 INFO:teuthology.orchestra.run.smithi063.stdout:nfs.foo.1.0.smithi174.pytptt smithi174 *:2049 running (2m) 14s ago 2m 75.7M - 4.0 e1d6a67b021e f220dbb449a3 2024-03-26T20:58:13.875 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.smithi063 smithi063 *:9100 running (8m) 2m ago 8m 15.9M - 1dbe0e931976 17d79ef02892 2024-03-26T20:58:13.875 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.smithi174 smithi174 *:9100 running (7m) 14s ago 7m 19.1M - 1dbe0e931976 6582440a94dc 2024-03-26T20:58:13.875 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0 smithi063 running (7m) 2m ago 7m 42.7M 3216M 17.2.0 e1d6a67b021e 761b1388d8fa 2024-03-26T20:58:13.875 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1 smithi063 running (6m) 2m ago 6m 42.2M 3216M 17.2.0 e1d6a67b021e 108275c17cad 2024-03-26T20:58:13.875 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2 smithi063 running (6m) 2m ago 6m 40.3M 3216M 17.2.0 e1d6a67b021e 107f685ad08d 2024-03-26T20:58:13.875 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3 smithi063 running (5m) 2m ago 5m 42.7M 3216M 17.2.0 e1d6a67b021e ab0f1a855957 2024-03-26T20:58:13.876 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4 smithi174 running (5m) 14s ago 5m 242M 3984M 17.2.0 e1d6a67b021e a0e877ecc06d 2024-03-26T20:58:13.876 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5 smithi174 running (4m) 14s ago 4m 280M 3984M 17.2.0 e1d6a67b021e 378c2aca7e71 2024-03-26T20:58:13.876 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6 smithi174 running (4m) 14s ago 4m 249M 3984M 17.2.0 e1d6a67b021e ffbb5b7ad3b6 2024-03-26T20:58:13.876 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7 smithi174 running (4m) 14s ago 4m 346M 3984M 17.2.0 e1d6a67b021e c1d57fcabb3f 2024-03-26T20:58:13.876 INFO:teuthology.orchestra.run.smithi063.stdout:prometheus.smithi063 smithi063 *:9095 running (7m) 2m ago 7m 60.4M - 514e6a882f6e a93dcc31497d 2024-03-26T20:58:14.438 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:14 smithi063 ceph-mon[24818]: from='client.14686 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:58:14.438 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:14 smithi063 ceph-mon[24818]: from='client.14690 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:58:14.438 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:14 smithi063 ceph-mon[24818]: Standby manager daemon smithi063.lxhujt started 2024-03-26T20:58:14.438 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:14 smithi063 ceph-mon[24818]: from='mgr.? 172.21.15.63:0/2810918224' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi063.lxhujt/crt"}]: dispatch 2024-03-26T20:58:14.438 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:14 smithi063 ceph-mon[24818]: from='mgr.? 172.21.15.63:0/2810918224' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-26T20:58:14.438 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:14 smithi063 ceph-mon[24818]: from='mgr.? 172.21.15.63:0/2810918224' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi063.lxhujt/key"}]: dispatch 2024-03-26T20:58:14.438 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:14 smithi063 ceph-mon[24818]: from='mgr.? 172.21.15.63:0/2810918224' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-26T20:58:14.460 INFO:teuthology.orchestra.run.smithi063.stdout: 5 74633 14.63 MB/sec execute 8 sec latency 44.613 ms 2024-03-26T20:58:14.644 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-03-26T20:58:14.644 INFO:teuthology.orchestra.run.smithi063.stdout: "mon": { 2024-03-26T20:58:14.644 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-26T20:58:14.644 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T20:58:14.644 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr": { 2024-03-26T20:58:14.644 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-03-26T20:58:14.644 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 1 2024-03-26T20:58:14.644 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T20:58:14.644 INFO:teuthology.orchestra.run.smithi063.stdout: "osd": { 2024-03-26T20:58:14.644 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-03-26T20:58:14.644 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T20:58:14.645 INFO:teuthology.orchestra.run.smithi063.stdout: "mds": { 2024-03-26T20:58:14.645 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-26T20:58:14.645 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T20:58:14.645 INFO:teuthology.orchestra.run.smithi063.stdout: "overall": { 2024-03-26T20:58:14.645 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-03-26T20:58:14.645 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 1 2024-03-26T20:58:14.645 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-03-26T20:58:14.645 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-03-26T20:58:14.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:14 smithi174 ceph-mon[26934]: from='client.14686 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:58:14.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:14 smithi174 ceph-mon[26934]: from='client.14690 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:58:14.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:14 smithi174 ceph-mon[26934]: Standby manager daemon smithi063.lxhujt started 2024-03-26T20:58:14.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:14 smithi174 ceph-mon[26934]: from='mgr.? 172.21.15.63:0/2810918224' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi063.lxhujt/crt"}]: dispatch 2024-03-26T20:58:14.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:14 smithi174 ceph-mon[26934]: from='mgr.? 172.21.15.63:0/2810918224' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-26T20:58:14.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:14 smithi174 ceph-mon[26934]: from='mgr.? 172.21.15.63:0/2810918224' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi063.lxhujt/key"}]: dispatch 2024-03-26T20:58:14.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:14 smithi174 ceph-mon[26934]: from='mgr.? 172.21.15.63:0/2810918224' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-26T20:58:15.240 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-03-26T20:58:15.241 INFO:teuthology.orchestra.run.smithi063.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d", 2024-03-26T20:58:15.241 INFO:teuthology.orchestra.run.smithi063.stdout: "in_progress": true, 2024-03-26T20:58:15.241 INFO:teuthology.orchestra.run.smithi063.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-26T20:58:15.241 INFO:teuthology.orchestra.run.smithi063.stdout: "services_complete": [], 2024-03-26T20:58:15.241 INFO:teuthology.orchestra.run.smithi063.stdout: "progress": "1/23 daemons upgraded", 2024-03-26T20:58:15.241 INFO:teuthology.orchestra.run.smithi063.stdout: "message": "", 2024-03-26T20:58:15.241 INFO:teuthology.orchestra.run.smithi063.stdout: "is_paused": false 2024-03-26T20:58:15.241 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-03-26T20:58:15.446 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:15 smithi063 ceph-mon[24818]: from='client.14694 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:58:15.447 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:15 smithi063 ceph-mon[24818]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "mgr metadata", "who": "smithi063.lxhujt", "id": "smithi063.lxhujt"}]: dispatch 2024-03-26T20:58:15.447 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:15 smithi063 ceph-mon[24818]: mgrmap e26: smithi174.sexoin(active, since 9s), standbys: smithi063.lxhujt 2024-03-26T20:58:15.447 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:15 smithi063 ceph-mon[24818]: from='client.? 172.21.15.63:0/2475023289' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T20:58:15.447 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:15 smithi063 ceph-mon[24818]: pgmap v8: 97 pgs: 97 active+clean; 329 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 4.3 MiB/s wr, 419 op/s 2024-03-26T20:58:15.460 INFO:teuthology.orchestra.run.smithi063.stdout: 5 75154 14.45 MB/sec execute 9 sec latency 22.175 ms 2024-03-26T20:58:15.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:15 smithi174 ceph-mon[26934]: from='client.14694 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:58:15.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:15 smithi174 ceph-mon[26934]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "mgr metadata", "who": "smithi063.lxhujt", "id": "smithi063.lxhujt"}]: dispatch 2024-03-26T20:58:15.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:15 smithi174 ceph-mon[26934]: mgrmap e26: smithi174.sexoin(active, since 9s), standbys: smithi063.lxhujt 2024-03-26T20:58:15.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:15 smithi174 ceph-mon[26934]: from='client.? 172.21.15.63:0/2475023289' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T20:58:15.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:15 smithi174 ceph-mon[26934]: pgmap v8: 97 pgs: 97 active+clean; 329 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 4.3 MiB/s wr, 419 op/s 2024-03-26T20:58:15.969 INFO:teuthology.orchestra.run.smithi063.stdout:HEALTH_OK 2024-03-26T20:58:16.454 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:16 smithi063 ceph-mon[24818]: from='client.24493 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:58:16.455 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:16 smithi063 ceph-mon[24818]: from='client.? 172.21.15.63:0/840504612' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-26T20:58:16.460 INFO:teuthology.orchestra.run.smithi063.stdout: 5 75568 13.85 MB/sec execute 10 sec latency 29.082 ms 2024-03-26T20:58:16.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:16 smithi174 ceph-mon[26934]: from='client.24493 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:58:16.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:16 smithi174 ceph-mon[26934]: from='client.? 172.21.15.63:0/840504612' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-26T20:58:17.460 INFO:teuthology.orchestra.run.smithi063.stdout: 5 76039 13.44 MB/sec execute 11 sec latency 54.448 ms 2024-03-26T20:58:17.469 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:17 smithi063 ceph-mon[24818]: pgmap v9: 97 pgs: 97 active+clean; 326 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 28 KiB/s rd, 7.2 MiB/s wr, 671 op/s 2024-03-26T20:58:17.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:17 smithi174 ceph-mon[26934]: pgmap v9: 97 pgs: 97 active+clean; 326 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 28 KiB/s rd, 7.2 MiB/s wr, 671 op/s 2024-03-26T20:58:18.460 INFO:teuthology.orchestra.run.smithi063.stdout: 5 76698 13.63 MB/sec execute 12 sec latency 39.720 ms 2024-03-26T20:58:19.460 INFO:teuthology.orchestra.run.smithi063.stdout: 5 77469 14.39 MB/sec execute 13 sec latency 63.477 ms 2024-03-26T20:58:19.743 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:19 smithi063 ceph-mon[24818]: pgmap v10: 97 pgs: 97 active+clean; 326 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 6.6 MiB/s wr, 614 op/s 2024-03-26T20:58:19.744 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:19 smithi063 ceph-mon[24818]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:58:19.744 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:19 smithi063 ceph-mon[24818]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:58:20.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:19 smithi174 ceph-mon[26934]: pgmap v10: 97 pgs: 97 active+clean; 326 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 6.6 MiB/s wr, 614 op/s 2024-03-26T20:58:20.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:19 smithi174 ceph-mon[26934]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:58:20.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:19 smithi174 ceph-mon[26934]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:58:20.460 INFO:teuthology.orchestra.run.smithi063.stdout: 5 78121 15.03 MB/sec execute 14 sec latency 24.473 ms 2024-03-26T20:58:20.923 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:20 smithi063 ceph-mon[24818]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T20:58:21.050 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:20 smithi174 ceph-mon[26934]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T20:58:21.460 INFO:teuthology.orchestra.run.smithi063.stdout: 5 78775 15.27 MB/sec execute 15 sec latency 26.403 ms 2024-03-26T20:58:21.936 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:21 smithi063 ceph-mon[24818]: pgmap v11: 97 pgs: 97 active+clean; 326 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 6.6 MiB/s wr, 614 op/s 2024-03-26T20:58:21.936 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:21 smithi063 ceph-mon[24818]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:58:21.936 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:21 smithi063 ceph-mon[24818]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:58:21.936 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:21 smithi063 ceph-mon[24818]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-26T20:58:21.936 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:21 smithi063 ceph-mon[24818]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-26T20:58:21.936 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:21 smithi063 ceph-mon[24818]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-26T20:58:21.936 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:21 smithi063 ceph-mon[24818]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-26T20:58:21.936 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:21 smithi063 ceph-mon[24818]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-26T20:58:21.936 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:21 smithi063 ceph-mon[24818]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-26T20:58:21.937 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:21 smithi063 ceph-mon[24818]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-26T20:58:21.937 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:21 smithi063 ceph-mon[24818]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-26T20:58:21.937 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:21 smithi063 ceph-mon[24818]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:58:22.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:21 smithi174 ceph-mon[26934]: pgmap v11: 97 pgs: 97 active+clean; 326 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 6.6 MiB/s wr, 614 op/s 2024-03-26T20:58:22.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:21 smithi174 ceph-mon[26934]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:58:22.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:21 smithi174 ceph-mon[26934]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:58:22.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:21 smithi174 ceph-mon[26934]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-26T20:58:22.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:21 smithi174 ceph-mon[26934]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-26T20:58:22.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:21 smithi174 ceph-mon[26934]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-26T20:58:22.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:21 smithi174 ceph-mon[26934]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-26T20:58:22.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:21 smithi174 ceph-mon[26934]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-26T20:58:22.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:21 smithi174 ceph-mon[26934]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-26T20:58:22.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:21 smithi174 ceph-mon[26934]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-26T20:58:22.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:21 smithi174 ceph-mon[26934]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-26T20:58:22.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:21 smithi174 ceph-mon[26934]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:58:22.460 INFO:teuthology.orchestra.run.smithi063.stdout: 5 79338 15.04 MB/sec execute 16 sec latency 22.456 ms 2024-03-26T20:58:22.955 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:22 smithi063 ceph-mon[24818]: Adjusting osd_memory_target on smithi063 to 1936M 2024-03-26T20:58:22.956 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:22 smithi063 ceph-mon[24818]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:58:22.956 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:22 smithi063 ceph-mon[24818]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:58:23.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:22 smithi174 ceph-mon[26934]: Adjusting osd_memory_target on smithi063 to 1936M 2024-03-26T20:58:23.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:22 smithi174 ceph-mon[26934]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:58:23.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:22 smithi174 ceph-mon[26934]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:58:23.460 INFO:teuthology.orchestra.run.smithi063.stdout: 5 79871 14.53 MB/sec execute 17 sec latency 23.181 ms 2024-03-26T20:58:23.715 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:23 smithi063 ceph-mon[24818]: pgmap v12: 97 pgs: 97 active+clean; 330 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 10 MiB/s wr, 968 op/s 2024-03-26T20:58:23.716 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:23 smithi063 ceph-mon[24818]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:58:23.716 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:23 smithi063 ceph-mon[24818]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:58:24.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:23 smithi174 ceph-mon[26934]: pgmap v12: 97 pgs: 97 active+clean; 330 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 10 MiB/s wr, 968 op/s 2024-03-26T20:58:24.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:23 smithi174 ceph-mon[26934]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:58:24.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:23 smithi174 ceph-mon[26934]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:58:24.460 INFO:teuthology.orchestra.run.smithi063.stdout: 5 80362 14.70 MB/sec execute 18 sec latency 52.175 ms 2024-03-26T20:58:25.460 INFO:teuthology.orchestra.run.smithi063.stdout: 5 80896 14.72 MB/sec execute 19 sec latency 38.725 ms 2024-03-26T20:58:25.740 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:25 smithi063 ceph-mon[24818]: pgmap v13: 97 pgs: 97 active+clean; 330 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 8.4 KiB/s rd, 7.1 MiB/s wr, 654 op/s 2024-03-26T20:58:25.931 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:25 smithi174 ceph-mon[26934]: pgmap v13: 97 pgs: 97 active+clean; 330 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 8.4 KiB/s rd, 7.1 MiB/s wr, 654 op/s 2024-03-26T20:58:26.461 INFO:teuthology.orchestra.run.smithi063.stdout: 5 81319 14.72 MB/sec execute 20 sec latency 27.588 ms 2024-03-26T20:58:27.461 INFO:teuthology.orchestra.run.smithi063.stdout: 5 81767 14.66 MB/sec execute 21 sec latency 50.924 ms 2024-03-26T20:58:27.795 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:27 smithi063 ceph-mon[24818]: pgmap v14: 97 pgs: 97 active+clean; 325 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 10 MiB/s wr, 969 op/s 2024-03-26T20:58:28.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:27 smithi174 ceph-mon[26934]: pgmap v14: 97 pgs: 97 active+clean; 325 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 10 MiB/s wr, 969 op/s 2024-03-26T20:58:28.461 INFO:teuthology.orchestra.run.smithi063.stdout: 5 82226 14.57 MB/sec execute 22 sec latency 72.808 ms 2024-03-26T20:58:29.461 INFO:teuthology.orchestra.run.smithi063.stdout: 5 82575 14.32 MB/sec execute 23 sec latency 39.429 ms 2024-03-26T20:58:29.570 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:29 smithi063 ceph-mon[24818]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:58:29.570 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:29 smithi063 ceph-mon[24818]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:58:29.570 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:29 smithi063 ceph-mon[24818]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-26T20:58:29.570 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:29 smithi063 ceph-mon[24818]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-26T20:58:29.571 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:29 smithi063 ceph-mon[24818]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-26T20:58:29.571 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:29 smithi063 ceph-mon[24818]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-26T20:58:29.571 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:29 smithi063 ceph-mon[24818]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-26T20:58:29.571 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:29 smithi063 ceph-mon[24818]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-26T20:58:29.571 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:29 smithi063 ceph-mon[24818]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-26T20:58:29.571 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:29 smithi063 ceph-mon[24818]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-26T20:58:29.571 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:29 smithi063 ceph-mon[24818]: Adjusting osd_memory_target on smithi174 to 2704M 2024-03-26T20:58:29.571 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:29 smithi063 ceph-mon[24818]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:58:29.571 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:29 smithi063 ceph-mon[24818]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:58:29.571 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:29 smithi063 ceph-mon[24818]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T20:58:29.571 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:29 smithi063 ceph-mon[24818]: Updating smithi063:/etc/ceph/ceph.conf 2024-03-26T20:58:29.571 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:29 smithi063 ceph-mon[24818]: Updating smithi174:/etc/ceph/ceph.conf 2024-03-26T20:58:29.571 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:29 smithi063 ceph-mon[24818]: pgmap v15: 97 pgs: 97 active+clean; 325 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.8 MiB/s wr, 668 op/s 2024-03-26T20:58:29.571 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:29 smithi063 ceph-mon[24818]: Updating smithi174:/var/lib/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-26T20:58:29.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:29 smithi174 ceph-mon[26934]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:58:29.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:29 smithi174 ceph-mon[26934]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:58:29.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:29 smithi174 ceph-mon[26934]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-26T20:58:29.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:29 smithi174 ceph-mon[26934]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-26T20:58:29.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:29 smithi174 ceph-mon[26934]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-26T20:58:29.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:29 smithi174 ceph-mon[26934]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-26T20:58:29.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:29 smithi174 ceph-mon[26934]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-26T20:58:29.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:29 smithi174 ceph-mon[26934]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-26T20:58:29.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:29 smithi174 ceph-mon[26934]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-26T20:58:29.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:29 smithi174 ceph-mon[26934]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-26T20:58:29.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:29 smithi174 ceph-mon[26934]: Adjusting osd_memory_target on smithi174 to 2704M 2024-03-26T20:58:29.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:29 smithi174 ceph-mon[26934]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:58:29.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:29 smithi174 ceph-mon[26934]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:58:29.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:29 smithi174 ceph-mon[26934]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T20:58:29.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:29 smithi174 ceph-mon[26934]: Updating smithi063:/etc/ceph/ceph.conf 2024-03-26T20:58:29.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:29 smithi174 ceph-mon[26934]: Updating smithi174:/etc/ceph/ceph.conf 2024-03-26T20:58:29.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:29 smithi174 ceph-mon[26934]: pgmap v15: 97 pgs: 97 active+clean; 325 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.8 MiB/s wr, 668 op/s 2024-03-26T20:58:29.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:29 smithi174 ceph-mon[26934]: Updating smithi174:/var/lib/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-26T20:58:30.461 INFO:teuthology.orchestra.run.smithi063.stdout: 5 83015 14.08 MB/sec execute 24 sec latency 46.648 ms 2024-03-26T20:58:30.574 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:30 smithi063 ceph-mon[24818]: Updating smithi063:/var/lib/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-26T20:58:30.574 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:30 smithi063 ceph-mon[24818]: Updating smithi174:/etc/ceph/ceph.client.admin.keyring 2024-03-26T20:58:30.574 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:30 smithi063 ceph-mon[24818]: Updating smithi063:/etc/ceph/ceph.client.admin.keyring 2024-03-26T20:58:30.574 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:30 smithi063 ceph-mon[24818]: Updating smithi174:/var/lib/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-26T20:58:30.574 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:30 smithi063 ceph-mon[24818]: Updating smithi063:/var/lib/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-26T20:58:30.574 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:30 smithi063 ceph-mon[24818]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:58:30.574 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:30 smithi063 ceph-mon[24818]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:58:30.574 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:30 smithi063 ceph-mon[24818]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:58:30.574 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:30 smithi063 ceph-mon[24818]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:58:30.574 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:30 smithi063 ceph-mon[24818]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:58:30.575 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:30 smithi063 ceph-mon[24818]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:58:30.575 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:30 smithi063 ceph-mon[24818]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:58:30.575 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:30 smithi063 ceph-mon[24818]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:58:30.575 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:30 smithi063 ceph-mon[24818]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:58:30.575 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:30 smithi063 ceph-mon[24818]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:58:30.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:30 smithi174 ceph-mon[26934]: Updating smithi063:/var/lib/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-26T20:58:30.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:30 smithi174 ceph-mon[26934]: Updating smithi174:/etc/ceph/ceph.client.admin.keyring 2024-03-26T20:58:30.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:30 smithi174 ceph-mon[26934]: Updating smithi063:/etc/ceph/ceph.client.admin.keyring 2024-03-26T20:58:30.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:30 smithi174 ceph-mon[26934]: Updating smithi174:/var/lib/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-26T20:58:30.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:30 smithi174 ceph-mon[26934]: Updating smithi063:/var/lib/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-26T20:58:30.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:30 smithi174 ceph-mon[26934]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:58:30.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:30 smithi174 ceph-mon[26934]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:58:30.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:30 smithi174 ceph-mon[26934]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:58:30.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:30 smithi174 ceph-mon[26934]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:58:30.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:30 smithi174 ceph-mon[26934]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:58:30.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:30 smithi174 ceph-mon[26934]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:58:30.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:30 smithi174 ceph-mon[26934]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:58:30.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:30 smithi174 ceph-mon[26934]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:58:30.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:30 smithi174 ceph-mon[26934]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:58:30.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:30 smithi174 ceph-mon[26934]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:58:31.461 INFO:teuthology.orchestra.run.smithi063.stdout: 5 83542 13.90 MB/sec execute 25 sec latency 58.229 ms 2024-03-26T20:58:31.588 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:31 smithi063 ceph-mon[24818]: Reconfiguring alertmanager.smithi063 (dependencies changed)... 2024-03-26T20:58:31.588 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:31 smithi063 ceph-mon[24818]: Reconfiguring daemon alertmanager.smithi063 on smithi063 2024-03-26T20:58:31.588 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:31 smithi063 ceph-mon[24818]: pgmap v16: 97 pgs: 97 active+clean; 325 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.8 MiB/s wr, 668 op/s 2024-03-26T20:58:32.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:31 smithi174 ceph-mon[26934]: Reconfiguring alertmanager.smithi063 (dependencies changed)... 2024-03-26T20:58:32.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:31 smithi174 ceph-mon[26934]: Reconfiguring daemon alertmanager.smithi063 on smithi063 2024-03-26T20:58:32.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:31 smithi174 ceph-mon[26934]: pgmap v16: 97 pgs: 97 active+clean; 325 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.8 MiB/s wr, 668 op/s 2024-03-26T20:58:32.461 INFO:teuthology.orchestra.run.smithi063.stdout: 5 84075 14.01 MB/sec execute 26 sec latency 39.185 ms 2024-03-26T20:58:33.461 INFO:teuthology.orchestra.run.smithi063.stdout: 5 84670 14.13 MB/sec execute 27 sec latency 23.047 ms 2024-03-26T20:58:33.598 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:33 smithi063 ceph-mon[24818]: pgmap v17: 97 pgs: 97 active+clean; 325 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 9.5 MiB/s wr, 925 op/s 2024-03-26T20:58:33.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:33 smithi174 ceph-mon[26934]: pgmap v17: 97 pgs: 97 active+clean; 325 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 9.5 MiB/s wr, 925 op/s 2024-03-26T20:58:34.461 INFO:teuthology.orchestra.run.smithi063.stdout: 5 85271 14.40 MB/sec execute 28 sec latency 26.527 ms 2024-03-26T20:58:35.466 INFO:teuthology.orchestra.run.smithi063.stdout: 5 85759 14.33 MB/sec execute 29 sec latency 32.546 ms 2024-03-26T20:58:35.873 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:35 smithi063 ceph-mon[24818]: pgmap v18: 97 pgs: 97 active+clean; 325 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.5 KiB/s rd, 5.7 MiB/s wr, 572 op/s 2024-03-26T20:58:35.874 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:35 smithi063 ceph-mon[24818]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T20:58:36.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:35 smithi174 ceph-mon[26934]: pgmap v18: 97 pgs: 97 active+clean; 325 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.5 KiB/s rd, 5.7 MiB/s wr, 572 op/s 2024-03-26T20:58:36.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:35 smithi174 ceph-mon[26934]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T20:58:36.461 INFO:teuthology.orchestra.run.smithi063.stdout: 5 86147 14.17 MB/sec execute 30 sec latency 51.578 ms 2024-03-26T20:58:37.461 INFO:teuthology.orchestra.run.smithi063.stdout: 5 86721 14.04 MB/sec execute 31 sec latency 39.389 ms 2024-03-26T20:58:37.897 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:37 smithi063 ceph-mon[24818]: pgmap v19: 97 pgs: 97 active+clean; 329 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 9.2 MiB/s wr, 893 op/s 2024-03-26T20:58:38.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:37 smithi174 ceph-mon[26934]: pgmap v19: 97 pgs: 97 active+clean; 329 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 9.2 MiB/s wr, 893 op/s 2024-03-26T20:58:38.462 INFO:teuthology.orchestra.run.smithi063.stdout: 5 87331 14.07 MB/sec execute 32 sec latency 29.650 ms 2024-03-26T20:58:39.461 INFO:teuthology.orchestra.run.smithi063.stdout: 5 87953 14.25 MB/sec execute 33 sec latency 28.883 ms 2024-03-26T20:58:39.917 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:39 smithi063 ceph-mon[24818]: pgmap v20: 97 pgs: 97 active+clean; 329 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.4 KiB/s rd, 6.2 MiB/s wr, 578 op/s 2024-03-26T20:58:40.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:39 smithi174 ceph-mon[26934]: pgmap v20: 97 pgs: 97 active+clean; 329 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.4 KiB/s rd, 6.2 MiB/s wr, 578 op/s 2024-03-26T20:58:40.461 INFO:teuthology.orchestra.run.smithi063.stdout: 5 88513 14.39 MB/sec execute 34 sec latency 33.689 ms 2024-03-26T20:58:41.461 INFO:teuthology.orchestra.run.smithi063.stdout: 5 89009 14.37 MB/sec execute 35 sec latency 25.744 ms 2024-03-26T20:58:41.848 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:41 smithi063 ceph-mon[24818]: pgmap v21: 97 pgs: 97 active+clean; 329 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.4 KiB/s rd, 6.2 MiB/s wr, 578 op/s 2024-03-26T20:58:42.051 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:41 smithi174 ceph-mon[26934]: pgmap v21: 97 pgs: 97 active+clean; 329 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.4 KiB/s rd, 6.2 MiB/s wr, 578 op/s 2024-03-26T20:58:42.461 INFO:teuthology.orchestra.run.smithi063.stdout: 5 89460 14.36 MB/sec execute 36 sec latency 28.554 ms 2024-03-26T20:58:43.461 INFO:teuthology.orchestra.run.smithi063.stdout: 5 89890 14.10 MB/sec execute 37 sec latency 55.624 ms 2024-03-26T20:58:43.635 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:43 smithi063 ceph-mon[24818]: pgmap v22: 97 pgs: 97 active+clean; 327 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.5 MiB/s wr, 904 op/s 2024-03-26T20:58:43.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:43 smithi174 ceph-mon[26934]: pgmap v22: 97 pgs: 97 active+clean; 327 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.5 MiB/s wr, 904 op/s 2024-03-26T20:58:44.461 INFO:teuthology.orchestra.run.smithi063.stdout: 5 90405 14.01 MB/sec execute 38 sec latency 61.717 ms 2024-03-26T20:58:45.462 INFO:teuthology.orchestra.run.smithi063.stdout: 5 90988 14.13 MB/sec execute 39 sec latency 22.352 ms 2024-03-26T20:58:45.913 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:45 smithi063 ceph-mon[24818]: pgmap v23: 97 pgs: 97 active+clean; 327 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 8.4 KiB/s rd, 6.8 MiB/s wr, 647 op/s 2024-03-26T20:58:46.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:45 smithi174 ceph-mon[26934]: pgmap v23: 97 pgs: 97 active+clean; 327 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 8.4 KiB/s rd, 6.8 MiB/s wr, 647 op/s 2024-03-26T20:58:46.462 INFO:teuthology.orchestra.run.smithi063.stdout: 5 91536 14.17 MB/sec execute 40 sec latency 30.425 ms 2024-03-26T20:58:46.519 INFO:teuthology.orchestra.run.smithi063.stdout:true 2024-03-26T20:58:47.461 INFO:teuthology.orchestra.run.smithi063.stdout: 5 91950 14.17 MB/sec execute 41 sec latency 57.043 ms 2024-03-26T20:58:47.573 INFO:teuthology.orchestra.run.smithi063.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-26T20:58:47.573 INFO:teuthology.orchestra.run.smithi063.stdout:alertmanager.smithi063 smithi063 *:9093,9094 running (7m) 28s ago 9m 27.9M - 0.23.0 ba2b418f427c f01f1c52094d 2024-03-26T20:58:47.574 INFO:teuthology.orchestra.run.smithi063.stdout:crash.smithi063 smithi063 running (9m) 28s ago 9m 6409k - 17.2.0 e1d6a67b021e 706c62f8af89 2024-03-26T20:58:47.574 INFO:teuthology.orchestra.run.smithi063.stdout:crash.smithi174 smithi174 running (8m) 25s ago 8m 6409k - 17.2.0 e1d6a67b021e 2c6bfe6d0ca6 2024-03-26T20:58:47.574 INFO:teuthology.orchestra.run.smithi063.stdout:grafana.smithi063 smithi063 *:3000 running (7m) 28s ago 8m 53.5M - 8.3.5 dad864ee21e9 74e19c64e1c4 2024-03-26T20:58:47.574 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foofs.smithi063.tzaznz smithi063 running (3m) 28s ago 3m 36.2M - 17.2.0 e1d6a67b021e 10a989fc7954 2024-03-26T20:58:47.574 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foofs.smithi174.gitdvk smithi174 running (3m) 25s ago 3m 11.8M - 17.2.0 e1d6a67b021e 4339a5d98d1d 2024-03-26T20:58:47.574 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.smithi063.lxhujt smithi063 *:9283 running (10m) 28s ago 10m 333M - 17.2.0 e1d6a67b021e 903406c7bde7 2024-03-26T20:58:47.574 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.smithi174.sexoin smithi174 *:8443,9283 running (53s) 25s ago 8m 488M - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 c7ed959f9018 2024-03-26T20:58:47.574 INFO:teuthology.orchestra.run.smithi063.stdout:mon.smithi063 smithi063 running (10m) 28s ago 10m 47.7M 2048M 17.2.0 e1d6a67b021e e6bd58b6c0f9 2024-03-26T20:58:47.574 INFO:teuthology.orchestra.run.smithi063.stdout:mon.smithi174 smithi174 running (8m) 25s ago 8m 39.8M 2048M 17.2.0 e1d6a67b021e 0f81c09f0c0a 2024-03-26T20:58:47.574 INFO:teuthology.orchestra.run.smithi063.stdout:nfs.foo.0.0.smithi063.fgmpwa smithi063 *:2049 running (2m) 28s ago 2m 256M - 4.0 e1d6a67b021e 3d6c0dd9ce51 2024-03-26T20:58:47.574 INFO:teuthology.orchestra.run.smithi063.stdout:nfs.foo.1.0.smithi174.pytptt smithi174 *:2049 running (2m) 25s ago 2m 75.9M - 4.0 e1d6a67b021e f220dbb449a3 2024-03-26T20:58:47.574 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.smithi063 smithi063 *:9100 running (8m) 28s ago 8m 16.2M - 1.3.1 1dbe0e931976 17d79ef02892 2024-03-26T20:58:47.574 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.smithi174 smithi174 *:9100 running (8m) 25s ago 8m 19.0M - 1.3.1 1dbe0e931976 6582440a94dc 2024-03-26T20:58:47.574 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0 smithi063 running (7m) 28s ago 7m 294M 1936M 17.2.0 e1d6a67b021e 761b1388d8fa 2024-03-26T20:58:47.574 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1 smithi063 running (7m) 28s ago 7m 237M 1936M 17.2.0 e1d6a67b021e 108275c17cad 2024-03-26T20:58:47.575 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2 smithi063 running (6m) 28s ago 6m 176M 1936M 17.2.0 e1d6a67b021e 107f685ad08d 2024-03-26T20:58:47.575 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3 smithi063 running (6m) 28s ago 6m 263M 1936M 17.2.0 e1d6a67b021e ab0f1a855957 2024-03-26T20:58:47.575 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4 smithi174 running (5m) 25s ago 5m 271M 2704M 17.2.0 e1d6a67b021e a0e877ecc06d 2024-03-26T20:58:47.575 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5 smithi174 running (5m) 25s ago 5m 309M 2704M 17.2.0 e1d6a67b021e 378c2aca7e71 2024-03-26T20:58:47.575 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6 smithi174 running (5m) 25s ago 5m 266M 2704M 17.2.0 e1d6a67b021e ffbb5b7ad3b6 2024-03-26T20:58:47.575 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7 smithi174 running (4m) 25s ago 4m 378M 2704M 17.2.0 e1d6a67b021e c1d57fcabb3f 2024-03-26T20:58:47.575 INFO:teuthology.orchestra.run.smithi063.stdout:prometheus.smithi063 smithi063 *:9095 running (7m) 28s ago 7m 66.9M - 2.33.4 514e6a882f6e a93dcc31497d 2024-03-26T20:58:47.927 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:47 smithi063 ceph-mon[24818]: from='client.14710 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:58:47.927 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:47 smithi063 ceph-mon[24818]: pgmap v24: 97 pgs: 97 active+clean; 323 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.8 MiB/s wr, 955 op/s 2024-03-26T20:58:47.927 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:47 smithi063 ceph-mon[24818]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:58:47.927 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:47 smithi063 ceph-mon[24818]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:58:47.927 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:47 smithi063 ceph-mon[24818]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:58:47.927 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:47 smithi063 ceph-mon[24818]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:58:47.927 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:47 smithi063 ceph-mon[24818]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-03-26T20:58:48.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:47 smithi174 ceph-mon[26934]: from='client.14710 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:58:48.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:47 smithi174 ceph-mon[26934]: pgmap v24: 97 pgs: 97 active+clean; 323 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.8 MiB/s wr, 955 op/s 2024-03-26T20:58:48.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:47 smithi174 ceph-mon[26934]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:58:48.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:47 smithi174 ceph-mon[26934]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:58:48.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:47 smithi174 ceph-mon[26934]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:58:48.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:47 smithi174 ceph-mon[26934]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:58:48.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:47 smithi174 ceph-mon[26934]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-03-26T20:58:48.297 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-03-26T20:58:48.297 INFO:teuthology.orchestra.run.smithi063.stdout: "mon": { 2024-03-26T20:58:48.297 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-26T20:58:48.298 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T20:58:48.298 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr": { 2024-03-26T20:58:48.298 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-03-26T20:58:48.298 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 1 2024-03-26T20:58:48.298 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T20:58:48.299 INFO:teuthology.orchestra.run.smithi063.stdout: "osd": { 2024-03-26T20:58:48.299 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-03-26T20:58:48.299 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T20:58:48.299 INFO:teuthology.orchestra.run.smithi063.stdout: "mds": { 2024-03-26T20:58:48.299 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-26T20:58:48.299 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T20:58:48.299 INFO:teuthology.orchestra.run.smithi063.stdout: "overall": { 2024-03-26T20:58:48.300 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-03-26T20:58:48.300 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 1 2024-03-26T20:58:48.300 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-03-26T20:58:48.300 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-03-26T20:58:48.462 INFO:teuthology.orchestra.run.smithi063.stdout: 5 92342 14.14 MB/sec execute 42 sec latency 34.506 ms 2024-03-26T20:58:48.852 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-03-26T20:58:48.852 INFO:teuthology.orchestra.run.smithi063.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d", 2024-03-26T20:58:48.852 INFO:teuthology.orchestra.run.smithi063.stdout: "in_progress": true, 2024-03-26T20:58:48.852 INFO:teuthology.orchestra.run.smithi063.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-26T20:58:48.852 INFO:teuthology.orchestra.run.smithi063.stdout: "services_complete": [], 2024-03-26T20:58:48.853 INFO:teuthology.orchestra.run.smithi063.stdout: "progress": "1/23 daemons upgraded", 2024-03-26T20:58:48.853 INFO:teuthology.orchestra.run.smithi063.stdout: "message": "", 2024-03-26T20:58:48.853 INFO:teuthology.orchestra.run.smithi063.stdout: "is_paused": false 2024-03-26T20:58:48.853 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-03-26T20:58:48.964 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:48 smithi063 ceph-mon[24818]: from='client.14714 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:58:48.965 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:48 smithi063 ceph-mon[24818]: from='client.14718 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:58:48.965 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:48 smithi063 ceph-mon[24818]: Reconfiguring grafana.smithi063 (dependencies changed)... 2024-03-26T20:58:48.965 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:48 smithi063 ceph-mon[24818]: Regenerating cephadm self-signed grafana TLS certificates 2024-03-26T20:58:48.965 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:48 smithi063 ceph-mon[24818]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-03-26T20:58:48.965 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:48 smithi063 ceph-mon[24818]: Reconfiguring daemon grafana.smithi063 on smithi063 2024-03-26T20:58:48.965 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:48 smithi063 ceph-mon[24818]: from='client.? 172.21.15.63:0/1042922557' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T20:58:49.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:48 smithi174 ceph-mon[26934]: from='client.14714 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:58:49.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:48 smithi174 ceph-mon[26934]: from='client.14718 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:58:49.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:48 smithi174 ceph-mon[26934]: Reconfiguring grafana.smithi063 (dependencies changed)... 2024-03-26T20:58:49.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:48 smithi174 ceph-mon[26934]: Regenerating cephadm self-signed grafana TLS certificates 2024-03-26T20:58:49.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:48 smithi174 ceph-mon[26934]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-03-26T20:58:49.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:48 smithi174 ceph-mon[26934]: Reconfiguring daemon grafana.smithi063 on smithi063 2024-03-26T20:58:49.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:48 smithi174 ceph-mon[26934]: from='client.? 172.21.15.63:0/1042922557' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T20:58:49.392 INFO:teuthology.orchestra.run.smithi063.stdout:HEALTH_OK 2024-03-26T20:58:49.462 INFO:teuthology.orchestra.run.smithi063.stdout: 5 92723 14.09 MB/sec execute 43 sec latency 36.017 ms 2024-03-26T20:58:49.970 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:49 smithi063 ceph-mon[24818]: pgmap v25: 97 pgs: 97 active+clean; 323 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 6.3 MiB/s wr, 634 op/s 2024-03-26T20:58:49.970 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:49 smithi063 ceph-mon[24818]: from='client.? 172.21.15.63:0/2272714928' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-26T20:58:50.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:49 smithi174 ceph-mon[26934]: pgmap v25: 97 pgs: 97 active+clean; 323 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 6.3 MiB/s wr, 634 op/s 2024-03-26T20:58:50.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:49 smithi174 ceph-mon[26934]: from='client.? 172.21.15.63:0/2272714928' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-26T20:58:50.462 INFO:teuthology.orchestra.run.smithi063.stdout: 5 93228 13.98 MB/sec execute 44 sec latency 36.346 ms 2024-03-26T20:58:50.981 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:50 smithi063 ceph-mon[24818]: from='client.24509 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:58:50.982 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:50 smithi063 ceph-mon[24818]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T20:58:51.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:50 smithi174 ceph-mon[26934]: from='client.24509 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:58:51.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:50 smithi174 ceph-mon[26934]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T20:58:51.462 INFO:teuthology.orchestra.run.smithi063.stdout: 5 93663 13.89 MB/sec execute 45 sec latency 40.035 ms 2024-03-26T20:58:51.747 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:51 smithi063 ceph-mon[24818]: pgmap v26: 97 pgs: 97 active+clean; 323 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 6.3 MiB/s wr, 634 op/s 2024-03-26T20:58:52.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:51 smithi174 ceph-mon[26934]: pgmap v26: 97 pgs: 97 active+clean; 323 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 6.3 MiB/s wr, 634 op/s 2024-03-26T20:58:52.462 INFO:teuthology.orchestra.run.smithi063.stdout: 5 94078 13.75 MB/sec execute 46 sec latency 43.115 ms 2024-03-26T20:58:53.462 INFO:teuthology.orchestra.run.smithi063.stdout: 5 94532 13.75 MB/sec execute 47 sec latency 36.950 ms 2024-03-26T20:58:53.528 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:53 smithi063 ceph-mon[24818]: pgmap v27: 97 pgs: 97 active+clean; 328 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 9.2 MiB/s wr, 896 op/s 2024-03-26T20:58:53.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:53 smithi174 ceph-mon[26934]: pgmap v27: 97 pgs: 97 active+clean; 328 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 9.2 MiB/s wr, 896 op/s 2024-03-26T20:58:54.462 INFO:teuthology.orchestra.run.smithi063.stdout: 5 95054 13.78 MB/sec execute 48 sec latency 36.521 ms 2024-03-26T20:58:55.462 INFO:teuthology.orchestra.run.smithi063.stdout: 5 95407 13.72 MB/sec execute 49 sec latency 41.494 ms 2024-03-26T20:58:55.805 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:55 smithi063 ceph-mon[24818]: pgmap v28: 97 pgs: 97 active+clean; 328 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.9 MiB/s wr, 570 op/s 2024-03-26T20:58:56.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:55 smithi174 ceph-mon[26934]: pgmap v28: 97 pgs: 97 active+clean; 328 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.9 MiB/s wr, 570 op/s 2024-03-26T20:58:56.462 INFO:teuthology.orchestra.run.smithi063.stdout: 5 95782 13.73 MB/sec execute 50 sec latency 38.786 ms 2024-03-26T20:58:57.462 INFO:teuthology.orchestra.run.smithi063.stdout: 5 96167 13.69 MB/sec execute 51 sec latency 47.149 ms 2024-03-26T20:58:57.869 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:57 smithi063 ceph-mon[24818]: pgmap v29: 97 pgs: 97 active+clean; 326 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.5 MiB/s wr, 824 op/s 2024-03-26T20:58:58.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:57 smithi174 ceph-mon[26934]: pgmap v29: 97 pgs: 97 active+clean; 326 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.5 MiB/s wr, 824 op/s 2024-03-26T20:58:58.462 INFO:teuthology.orchestra.run.smithi063.stdout: 5 96582 13.62 MB/sec execute 52 sec latency 35.838 ms 2024-03-26T20:58:59.462 INFO:teuthology.orchestra.run.smithi063.stdout: 5 97032 13.52 MB/sec execute 53 sec latency 37.576 ms 2024-03-26T20:58:59.906 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:58:59 smithi063 ceph-mon[24818]: pgmap v30: 97 pgs: 97 active+clean; 326 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.5 MiB/s wr, 516 op/s 2024-03-26T20:59:00.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:58:59 smithi174 ceph-mon[26934]: pgmap v30: 97 pgs: 97 active+clean; 326 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.5 MiB/s wr, 516 op/s 2024-03-26T20:59:00.462 INFO:teuthology.orchestra.run.smithi063.stdout: 5 97544 13.45 MB/sec execute 54 sec latency 52.597 ms 2024-03-26T20:59:01.462 INFO:teuthology.orchestra.run.smithi063.stdout: 5 97988 13.43 MB/sec execute 55 sec latency 46.463 ms 2024-03-26T20:59:01.922 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:01 smithi063 ceph-mon[24818]: pgmap v31: 97 pgs: 97 active+clean; 326 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.5 MiB/s wr, 516 op/s 2024-03-26T20:59:02.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:01 smithi174 ceph-mon[26934]: pgmap v31: 97 pgs: 97 active+clean; 326 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.5 MiB/s wr, 516 op/s 2024-03-26T20:59:02.462 INFO:teuthology.orchestra.run.smithi063.stdout: 5 98489 13.49 MB/sec execute 56 sec latency 31.986 ms 2024-03-26T20:59:03.445 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:03 smithi063 ceph-mon[24818]: pgmap v32: 97 pgs: 97 active+clean; 325 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.0 MiB/s wr, 782 op/s 2024-03-26T20:59:03.462 INFO:teuthology.orchestra.run.smithi063.stdout: 5 98919 13.46 MB/sec execute 57 sec latency 37.588 ms 2024-03-26T20:59:03.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:03 smithi174 ceph-mon[26934]: pgmap v32: 97 pgs: 97 active+clean; 325 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.0 MiB/s wr, 782 op/s 2024-03-26T20:59:04.462 INFO:teuthology.orchestra.run.smithi063.stdout: 5 99302 13.47 MB/sec execute 58 sec latency 38.602 ms 2024-03-26T20:59:05.462 INFO:teuthology.orchestra.run.smithi063.stdout: 5 99669 13.40 MB/sec execute 59 sec latency 38.413 ms 2024-03-26T20:59:05.733 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:05 smithi063 ceph-mon[24818]: pgmap v33: 97 pgs: 97 active+clean; 325 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.1 MiB/s wr, 519 op/s 2024-03-26T20:59:05.734 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:05 smithi063 ceph-mon[24818]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T20:59:06.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:05 smithi174 ceph-mon[26934]: pgmap v33: 97 pgs: 97 active+clean; 325 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.1 MiB/s wr, 519 op/s 2024-03-26T20:59:06.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:05 smithi174 ceph-mon[26934]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T20:59:06.462 INFO:teuthology.orchestra.run.smithi063.stdout: 5 100111 13.39 MB/sec execute 60 sec latency 35.101 ms 2024-03-26T20:59:07.463 INFO:teuthology.orchestra.run.smithi063.stdout: 5 100429 13.25 MB/sec execute 61 sec latency 60.467 ms 2024-03-26T20:59:07.900 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:07 smithi063 ceph-mon[24818]: pgmap v34: 97 pgs: 97 active+clean; 332 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 8.0 MiB/s wr, 770 op/s 2024-03-26T20:59:08.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:07 smithi174 ceph-mon[26934]: pgmap v34: 97 pgs: 97 active+clean; 332 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 8.0 MiB/s wr, 770 op/s 2024-03-26T20:59:08.462 INFO:teuthology.orchestra.run.smithi063.stdout: 5 100772 13.17 MB/sec execute 62 sec latency 53.501 ms 2024-03-26T20:59:09.462 INFO:teuthology.orchestra.run.smithi063.stdout: 5 101136 13.06 MB/sec execute 63 sec latency 52.285 ms 2024-03-26T20:59:09.925 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:09 smithi063 ceph-mon[24818]: pgmap v35: 97 pgs: 97 active+clean; 332 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.5 MiB/s wr, 516 op/s 2024-03-26T20:59:09.926 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:09 smithi063 ceph-mon[24818]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:59:09.926 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:09 smithi063 ceph-mon[24818]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:59:10.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:09 smithi174 ceph-mon[26934]: pgmap v35: 97 pgs: 97 active+clean; 332 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.5 MiB/s wr, 516 op/s 2024-03-26T20:59:10.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:09 smithi174 ceph-mon[26934]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:59:10.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:09 smithi174 ceph-mon[26934]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:59:10.462 INFO:teuthology.orchestra.run.smithi063.stdout: 5 101481 13.01 MB/sec execute 64 sec latency 37.576 ms 2024-03-26T20:59:10.944 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:10 smithi063 ceph-mon[24818]: Reconfiguring node-exporter.smithi063 (dependencies changed)... 2024-03-26T20:59:10.944 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:10 smithi063 ceph-mon[24818]: Reconfiguring daemon node-exporter.smithi063 on smithi063 2024-03-26T20:59:11.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:10 smithi174 ceph-mon[26934]: Reconfiguring node-exporter.smithi063 (dependencies changed)... 2024-03-26T20:59:11.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:10 smithi174 ceph-mon[26934]: Reconfiguring daemon node-exporter.smithi063 on smithi063 2024-03-26T20:59:11.462 INFO:teuthology.orchestra.run.smithi063.stdout: 5 101844 13.01 MB/sec execute 65 sec latency 47.896 ms 2024-03-26T20:59:11.952 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:11 smithi063 ceph-mon[24818]: pgmap v36: 97 pgs: 97 active+clean; 332 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.5 MiB/s wr, 516 op/s 2024-03-26T20:59:12.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:11 smithi174 ceph-mon[26934]: pgmap v36: 97 pgs: 97 active+clean; 332 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.5 MiB/s wr, 516 op/s 2024-03-26T20:59:12.462 INFO:teuthology.orchestra.run.smithi063.stdout: 5 102220 12.96 MB/sec execute 66 sec latency 64.262 ms 2024-03-26T20:59:13.462 INFO:teuthology.orchestra.run.smithi063.stdout: 5 102507 12.90 MB/sec execute 67 sec latency 45.068 ms 2024-03-26T20:59:13.475 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:13 smithi063 ceph-mon[24818]: pgmap v37: 97 pgs: 97 active+clean; 323 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.5 MiB/s wr, 740 op/s 2024-03-26T20:59:13.475 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:13 smithi063 ceph-mon[24818]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:59:13.475 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:13 smithi063 ceph-mon[24818]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:59:13.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:13 smithi174 ceph-mon[26934]: pgmap v37: 97 pgs: 97 active+clean; 323 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.5 MiB/s wr, 740 op/s 2024-03-26T20:59:13.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:13 smithi174 ceph-mon[26934]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:59:13.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:13 smithi174 ceph-mon[26934]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:59:14.462 INFO:teuthology.orchestra.run.smithi063.stdout: 5 102879 12.93 MB/sec execute 68 sec latency 33.619 ms 2024-03-26T20:59:14.482 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:14 smithi063 ceph-mon[24818]: Reconfiguring prometheus.smithi063 (dependencies changed)... 2024-03-26T20:59:14.483 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:14 smithi063 ceph-mon[24818]: Reconfiguring daemon prometheus.smithi063 on smithi063 2024-03-26T20:59:14.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:14 smithi174 ceph-mon[26934]: Reconfiguring prometheus.smithi063 (dependencies changed)... 2024-03-26T20:59:14.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:14 smithi174 ceph-mon[26934]: Reconfiguring daemon prometheus.smithi063 on smithi063 2024-03-26T20:59:15.463 INFO:teuthology.orchestra.run.smithi063.stdout: 5 103177 12.83 MB/sec execute 69 sec latency 58.783 ms 2024-03-26T20:59:15.496 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:15 smithi063 ceph-mon[24818]: pgmap v38: 97 pgs: 97 active+clean; 323 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.9 MiB/s wr, 475 op/s 2024-03-26T20:59:15.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:15 smithi174 ceph-mon[26934]: pgmap v38: 97 pgs: 97 active+clean; 323 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.9 MiB/s wr, 475 op/s 2024-03-26T20:59:16.463 INFO:teuthology.orchestra.run.smithi063.stdout: 5 103533 12.82 MB/sec execute 70 sec latency 39.431 ms 2024-03-26T20:59:17.463 INFO:teuthology.orchestra.run.smithi063.stdout: 5 103896 12.72 MB/sec execute 71 sec latency 35.706 ms 2024-03-26T20:59:17.770 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:17 smithi063 ceph-mon[24818]: pgmap v39: 97 pgs: 97 active+clean; 332 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 7.2 MiB/s wr, 674 op/s 2024-03-26T20:59:18.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:17 smithi174 ceph-mon[26934]: pgmap v39: 97 pgs: 97 active+clean; 332 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 7.2 MiB/s wr, 674 op/s 2024-03-26T20:59:18.463 INFO:teuthology.orchestra.run.smithi063.stdout: 5 104313 12.67 MB/sec execute 72 sec latency 41.456 ms 2024-03-26T20:59:19.469 INFO:teuthology.orchestra.run.smithi063.stdout: 5 104780 12.60 MB/sec execute 73 sec latency 59.267 ms 2024-03-26T20:59:19.789 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:19 smithi063 ceph-mon[24818]: pgmap v40: 97 pgs: 97 active+clean; 332 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.3 MiB/s wr, 422 op/s 2024-03-26T20:59:20.036 INFO:teuthology.orchestra.run.smithi063.stdout:true 2024-03-26T20:59:20.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:19 smithi174 ceph-mon[26934]: pgmap v40: 97 pgs: 97 active+clean; 332 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.3 MiB/s wr, 422 op/s 2024-03-26T20:59:20.469 INFO:teuthology.orchestra.run.smithi063.stdout: 5 105091 12.56 MB/sec execute 74 sec latency 36.392 ms 2024-03-26T20:59:20.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:20 smithi063 ceph-mon[24818]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T20:59:21.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:20 smithi174 ceph-mon[26934]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T20:59:21.154 INFO:teuthology.orchestra.run.smithi063.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-26T20:59:21.154 INFO:teuthology.orchestra.run.smithi063.stdout:alertmanager.smithi063 smithi063 *:9093,9094 starting - - - - 2024-03-26T20:59:21.154 INFO:teuthology.orchestra.run.smithi063.stdout:crash.smithi063 smithi063 running (9m) 61s ago 9m 6409k - 17.2.0 e1d6a67b021e 706c62f8af89 2024-03-26T20:59:21.154 INFO:teuthology.orchestra.run.smithi063.stdout:crash.smithi174 smithi174 running (9m) 59s ago 9m 6409k - 17.2.0 e1d6a67b021e 2c6bfe6d0ca6 2024-03-26T20:59:21.155 INFO:teuthology.orchestra.run.smithi063.stdout:grafana.smithi063 smithi063 *:3000 starting - - - - 2024-03-26T20:59:21.155 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foofs.smithi063.tzaznz smithi063 running (3m) 61s ago 3m 36.2M - 17.2.0 e1d6a67b021e 10a989fc7954 2024-03-26T20:59:21.155 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foofs.smithi174.gitdvk smithi174 running (3m) 59s ago 3m 11.8M - 17.2.0 e1d6a67b021e 4339a5d98d1d 2024-03-26T20:59:21.155 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.smithi063.lxhujt smithi063 *:9283 running (10m) 61s ago 10m 333M - 17.2.0 e1d6a67b021e 903406c7bde7 2024-03-26T20:59:21.155 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.smithi174.sexoin smithi174 *:8443,9283 running (86s) 59s ago 9m 488M - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 c7ed959f9018 2024-03-26T20:59:21.155 INFO:teuthology.orchestra.run.smithi063.stdout:mon.smithi063 smithi063 running (11m) 61s ago 11m 47.7M 2048M 17.2.0 e1d6a67b021e e6bd58b6c0f9 2024-03-26T20:59:21.156 INFO:teuthology.orchestra.run.smithi063.stdout:mon.smithi174 smithi174 running (8m) 59s ago 8m 39.8M 2048M 17.2.0 e1d6a67b021e 0f81c09f0c0a 2024-03-26T20:59:21.156 INFO:teuthology.orchestra.run.smithi063.stdout:nfs.foo.0.0.smithi063.fgmpwa smithi063 *:2049 running (3m) 61s ago 3m 256M - 4.0 e1d6a67b021e 3d6c0dd9ce51 2024-03-26T20:59:21.156 INFO:teuthology.orchestra.run.smithi063.stdout:nfs.foo.1.0.smithi174.pytptt smithi174 *:2049 running (3m) 59s ago 3m 75.9M - 4.0 e1d6a67b021e f220dbb449a3 2024-03-26T20:59:21.156 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.smithi063 smithi063 *:9100 starting - - - - 2024-03-26T20:59:21.156 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.smithi174 smithi174 *:9100 running (8m) 59s ago 8m 19.0M - 1.3.1 1dbe0e931976 6582440a94dc 2024-03-26T20:59:21.156 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0 smithi063 running (8m) 61s ago 8m 294M 1936M 17.2.0 e1d6a67b021e 761b1388d8fa 2024-03-26T20:59:21.156 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1 smithi063 running (7m) 61s ago 7m 237M 1936M 17.2.0 e1d6a67b021e 108275c17cad 2024-03-26T20:59:21.157 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2 smithi063 running (7m) 61s ago 7m 176M 1936M 17.2.0 e1d6a67b021e 107f685ad08d 2024-03-26T20:59:21.157 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3 smithi063 running (6m) 61s ago 6m 263M 1936M 17.2.0 e1d6a67b021e ab0f1a855957 2024-03-26T20:59:21.157 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4 smithi174 running (6m) 59s ago 6m 271M 2704M 17.2.0 e1d6a67b021e a0e877ecc06d 2024-03-26T20:59:21.157 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5 smithi174 running (6m) 59s ago 5m 309M 2704M 17.2.0 e1d6a67b021e 378c2aca7e71 2024-03-26T20:59:21.157 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6 smithi174 running (5m) 59s ago 5m 266M 2704M 17.2.0 e1d6a67b021e ffbb5b7ad3b6 2024-03-26T20:59:21.157 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7 smithi174 running (5m) 59s ago 5m 378M 2704M 17.2.0 e1d6a67b021e c1d57fcabb3f 2024-03-26T20:59:21.158 INFO:teuthology.orchestra.run.smithi063.stdout:prometheus.smithi063 smithi063 *:9095 running (8m) 61s ago 8m 66.9M - 2.33.4 514e6a882f6e a93dcc31497d 2024-03-26T20:59:21.469 INFO:teuthology.orchestra.run.smithi063.stdout: 5 105467 12.56 MB/sec execute 75 sec latency 54.441 ms 2024-03-26T20:59:21.821 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:21 smithi063 ceph-mon[24818]: from='client.14734 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:59:21.822 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:21 smithi063 ceph-mon[24818]: from='client.14738 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:59:21.822 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:21 smithi063 ceph-mon[24818]: pgmap v41: 97 pgs: 97 active+clean; 332 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.3 MiB/s wr, 422 op/s 2024-03-26T20:59:21.884 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-03-26T20:59:21.885 INFO:teuthology.orchestra.run.smithi063.stdout: "mon": { 2024-03-26T20:59:21.885 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-26T20:59:21.885 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T20:59:21.885 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr": { 2024-03-26T20:59:21.885 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-03-26T20:59:21.885 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 1 2024-03-26T20:59:21.885 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T20:59:21.885 INFO:teuthology.orchestra.run.smithi063.stdout: "osd": { 2024-03-26T20:59:21.885 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-03-26T20:59:21.885 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T20:59:21.885 INFO:teuthology.orchestra.run.smithi063.stdout: "mds": { 2024-03-26T20:59:21.886 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-26T20:59:21.886 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T20:59:21.886 INFO:teuthology.orchestra.run.smithi063.stdout: "overall": { 2024-03-26T20:59:21.886 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-03-26T20:59:21.886 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 1 2024-03-26T20:59:21.886 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-03-26T20:59:21.886 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-03-26T20:59:22.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:21 smithi174 ceph-mon[26934]: from='client.14734 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:59:22.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:21 smithi174 ceph-mon[26934]: from='client.14738 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:59:22.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:21 smithi174 ceph-mon[26934]: pgmap v41: 97 pgs: 97 active+clean; 332 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.3 MiB/s wr, 422 op/s 2024-03-26T20:59:22.424 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-03-26T20:59:22.424 INFO:teuthology.orchestra.run.smithi063.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d", 2024-03-26T20:59:22.424 INFO:teuthology.orchestra.run.smithi063.stdout: "in_progress": true, 2024-03-26T20:59:22.424 INFO:teuthology.orchestra.run.smithi063.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-26T20:59:22.424 INFO:teuthology.orchestra.run.smithi063.stdout: "services_complete": [], 2024-03-26T20:59:22.424 INFO:teuthology.orchestra.run.smithi063.stdout: "progress": "1/23 daemons upgraded", 2024-03-26T20:59:22.424 INFO:teuthology.orchestra.run.smithi063.stdout: "message": "", 2024-03-26T20:59:22.424 INFO:teuthology.orchestra.run.smithi063.stdout: "is_paused": false 2024-03-26T20:59:22.425 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-03-26T20:59:22.469 INFO:teuthology.orchestra.run.smithi063.stdout: 5 105915 12.55 MB/sec execute 76 sec latency 32.051 ms 2024-03-26T20:59:22.835 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:22 smithi063 ceph-mon[24818]: from='client.14742 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:59:22.836 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:22 smithi063 ceph-mon[24818]: from='client.? 172.21.15.63:0/1959893148' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T20:59:23.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:22 smithi174 ceph-mon[26934]: from='client.14742 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:59:23.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:22 smithi174 ceph-mon[26934]: from='client.? 172.21.15.63:0/1959893148' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T20:59:23.069 INFO:teuthology.orchestra.run.smithi063.stdout:HEALTH_OK 2024-03-26T20:59:23.469 INFO:teuthology.orchestra.run.smithi063.stdout: 5 106239 12.57 MB/sec execute 77 sec latency 37.915 ms 2024-03-26T20:59:23.842 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:23 smithi063 ceph-mon[24818]: from='client.14750 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:59:23.842 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:23 smithi063 ceph-mon[24818]: pgmap v42: 97 pgs: 97 active+clean; 325 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.5 MiB/s wr, 664 op/s 2024-03-26T20:59:23.842 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:23 smithi063 ceph-mon[24818]: from='client.? 172.21.15.63:0/1105829061' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-26T20:59:24.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:23 smithi174 ceph-mon[26934]: from='client.14750 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:59:24.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:23 smithi174 ceph-mon[26934]: pgmap v42: 97 pgs: 97 active+clean; 325 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.5 MiB/s wr, 664 op/s 2024-03-26T20:59:24.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:23 smithi174 ceph-mon[26934]: from='client.? 172.21.15.63:0/1105829061' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-26T20:59:24.469 INFO:teuthology.orchestra.run.smithi063.stdout: 5 106601 12.54 MB/sec execute 78 sec latency 38.720 ms 2024-03-26T20:59:25.469 INFO:teuthology.orchestra.run.smithi063.stdout: 5 106914 12.48 MB/sec execute 79 sec latency 62.397 ms 2024-03-26T20:59:25.888 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:25 smithi063 ceph-mon[24818]: pgmap v43: 97 pgs: 97 active+clean; 325 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.5 MiB/s wr, 440 op/s 2024-03-26T20:59:26.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:25 smithi174 ceph-mon[26934]: pgmap v43: 97 pgs: 97 active+clean; 325 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.5 MiB/s wr, 440 op/s 2024-03-26T20:59:26.469 INFO:teuthology.orchestra.run.smithi063.stdout: 5 107420 12.47 MB/sec execute 80 sec latency 22.503 ms 2024-03-26T20:59:27.469 INFO:teuthology.orchestra.run.smithi063.stdout: 5 107964 12.45 MB/sec execute 81 sec latency 35.061 ms 2024-03-26T20:59:27.915 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:27 smithi063 ceph-mon[24818]: pgmap v44: 97 pgs: 97 active+clean; 330 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.4 KiB/s rd, 6.9 MiB/s wr, 653 op/s 2024-03-26T20:59:28.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:27 smithi174 ceph-mon[26934]: pgmap v44: 97 pgs: 97 active+clean; 330 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.4 KiB/s rd, 6.9 MiB/s wr, 653 op/s 2024-03-26T20:59:28.469 INFO:teuthology.orchestra.run.smithi063.stdout: 5 108515 12.48 MB/sec execute 82 sec latency 37.489 ms 2024-03-26T20:59:29.469 INFO:teuthology.orchestra.run.smithi063.stdout: 5 109219 12.57 MB/sec execute 83 sec latency 45.481 ms 2024-03-26T20:59:29.949 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:29 smithi063 ceph-mon[24818]: pgmap v45: 97 pgs: 97 active+clean; 330 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.6 MiB/s wr, 454 op/s 2024-03-26T20:59:30.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:29 smithi174 ceph-mon[26934]: pgmap v45: 97 pgs: 97 active+clean; 330 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.6 MiB/s wr, 454 op/s 2024-03-26T20:59:30.469 INFO:teuthology.orchestra.run.smithi063.stdout: 5 109709 12.61 MB/sec execute 84 sec latency 28.121 ms 2024-03-26T20:59:31.469 INFO:teuthology.orchestra.run.smithi063.stdout: 5 110191 12.64 MB/sec execute 85 sec latency 27.493 ms 2024-03-26T20:59:31.965 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:31 smithi063 ceph-mon[24818]: pgmap v46: 97 pgs: 97 active+clean; 330 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.6 MiB/s wr, 454 op/s 2024-03-26T20:59:32.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:31 smithi174 ceph-mon[26934]: pgmap v46: 97 pgs: 97 active+clean; 330 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.6 MiB/s wr, 454 op/s 2024-03-26T20:59:32.469 INFO:teuthology.orchestra.run.smithi063.stdout: 5 110651 12.66 MB/sec execute 86 sec latency 42.210 ms 2024-03-26T20:59:33.469 INFO:teuthology.orchestra.run.smithi063.stdout: 5 110983 12.57 MB/sec execute 87 sec latency 38.832 ms 2024-03-26T20:59:33.494 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:33 smithi063 ceph-mon[24818]: pgmap v47: 97 pgs: 97 active+clean; 330 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.8 MiB/s wr, 786 op/s 2024-03-26T20:59:33.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:33 smithi174 ceph-mon[26934]: pgmap v47: 97 pgs: 97 active+clean; 330 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.8 MiB/s wr, 786 op/s 2024-03-26T20:59:34.469 INFO:teuthology.orchestra.run.smithi063.stdout: 5 111299 12.50 MB/sec execute 88 sec latency 49.125 ms 2024-03-26T20:59:35.469 INFO:teuthology.orchestra.run.smithi063.stdout: 5 111675 12.42 MB/sec execute 89 sec latency 85.320 ms 2024-03-26T20:59:35.908 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:35 smithi063 ceph-mon[24818]: pgmap v48: 97 pgs: 97 active+clean; 330 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.7 MiB/s wr, 544 op/s 2024-03-26T20:59:35.909 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:35 smithi063 ceph-mon[24818]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T20:59:36.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:35 smithi174 ceph-mon[26934]: pgmap v48: 97 pgs: 97 active+clean; 330 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.7 MiB/s wr, 544 op/s 2024-03-26T20:59:36.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:35 smithi174 ceph-mon[26934]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T20:59:36.469 INFO:teuthology.orchestra.run.smithi063.stdout: 5 112054 12.40 MB/sec execute 90 sec latency 35.298 ms 2024-03-26T20:59:37.472 INFO:teuthology.orchestra.run.smithi063.stdout: 5 112389 12.44 MB/sec execute 91 sec latency 53.405 ms 2024-03-26T20:59:37.752 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:37 smithi063 ceph-mon[24818]: pgmap v49: 97 pgs: 97 active+clean; 325 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 8.0 MiB/s wr, 788 op/s 2024-03-26T20:59:37.753 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:37 smithi063 ceph-mon[24818]: mgrmap e27: smithi174.sexoin(active, since 92s), standbys: smithi063.lxhujt 2024-03-26T20:59:37.753 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:37 smithi063 ceph-mon[24818]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:59:37.753 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:37 smithi063 ceph-mon[24818]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:59:38.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:37 smithi174 ceph-mon[26934]: pgmap v49: 97 pgs: 97 active+clean; 325 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 8.0 MiB/s wr, 788 op/s 2024-03-26T20:59:38.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:37 smithi174 ceph-mon[26934]: mgrmap e27: smithi174.sexoin(active, since 92s), standbys: smithi063.lxhujt 2024-03-26T20:59:38.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:37 smithi174 ceph-mon[26934]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:59:38.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:37 smithi174 ceph-mon[26934]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:59:38.472 INFO:teuthology.orchestra.run.smithi063.stdout: 5 112798 12.40 MB/sec execute 92 sec latency 56.120 ms 2024-03-26T20:59:38.775 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:38 smithi063 ceph-mon[24818]: Reconfiguring node-exporter.smithi174 (dependencies changed)... 2024-03-26T20:59:38.775 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:38 smithi063 ceph-mon[24818]: Reconfiguring daemon node-exporter.smithi174 on smithi174 2024-03-26T20:59:39.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:38 smithi174 ceph-mon[26934]: Reconfiguring node-exporter.smithi174 (dependencies changed)... 2024-03-26T20:59:39.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:38 smithi174 ceph-mon[26934]: Reconfiguring daemon node-exporter.smithi174 on smithi174 2024-03-26T20:59:39.472 INFO:teuthology.orchestra.run.smithi063.stdout: 5 113180 12.38 MB/sec execute 93 sec latency 47.012 ms 2024-03-26T20:59:39.793 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:39 smithi063 ceph-mon[24818]: pgmap v50: 97 pgs: 97 active+clean; 325 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.5 MiB/s wr, 575 op/s 2024-03-26T20:59:39.793 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:39 smithi063 ceph-mon[24818]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:59:39.793 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:39 smithi063 ceph-mon[24818]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:59:39.793 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:39 smithi063 ceph-mon[24818]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-26T20:59:39.793 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:39 smithi063 ceph-mon[24818]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi063.front.sepia.ceph.com:9093"}]: dispatch 2024-03-26T20:59:39.793 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:39 smithi063 ceph-mon[24818]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:59:39.793 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:39 smithi063 ceph-mon[24818]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-26T20:59:39.794 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:39 smithi063 ceph-mon[24818]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi063.front.sepia.ceph.com:3000"}]: dispatch 2024-03-26T20:59:39.794 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:39 smithi063 ceph-mon[24818]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:59:39.794 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:39 smithi063 ceph-mon[24818]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-26T20:59:39.794 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:39 smithi063 ceph-mon[24818]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi063.front.sepia.ceph.com:9095"}]: dispatch 2024-03-26T20:59:39.794 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:39 smithi063 ceph-mon[24818]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:59:39.794 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:39 smithi063 ceph-mon[24818]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T20:59:40.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:39 smithi174 ceph-mon[26934]: pgmap v50: 97 pgs: 97 active+clean; 325 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.5 MiB/s wr, 575 op/s 2024-03-26T20:59:40.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:39 smithi174 ceph-mon[26934]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:59:40.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:39 smithi174 ceph-mon[26934]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:59:40.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:39 smithi174 ceph-mon[26934]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-26T20:59:40.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:39 smithi174 ceph-mon[26934]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi063.front.sepia.ceph.com:9093"}]: dispatch 2024-03-26T20:59:40.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:39 smithi174 ceph-mon[26934]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:59:40.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:39 smithi174 ceph-mon[26934]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-26T20:59:40.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:39 smithi174 ceph-mon[26934]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi063.front.sepia.ceph.com:3000"}]: dispatch 2024-03-26T20:59:40.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:39 smithi174 ceph-mon[26934]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:59:40.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:39 smithi174 ceph-mon[26934]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-26T20:59:40.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:39 smithi174 ceph-mon[26934]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi063.front.sepia.ceph.com:9095"}]: dispatch 2024-03-26T20:59:40.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:39 smithi174 ceph-mon[26934]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:59:40.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:39 smithi174 ceph-mon[26934]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T20:59:40.472 INFO:teuthology.orchestra.run.smithi063.stdout: 5 113485 12.37 MB/sec execute 94 sec latency 49.786 ms 2024-03-26T20:59:40.807 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:40 smithi063 ceph-mon[24818]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-26T20:59:40.808 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:40 smithi063 ceph-mon[24818]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi063.front.sepia.ceph.com:9093"}]: dispatch 2024-03-26T20:59:40.808 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:40 smithi063 ceph-mon[24818]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-26T20:59:40.808 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:40 smithi063 ceph-mon[24818]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi063.front.sepia.ceph.com:3000"}]: dispatch 2024-03-26T20:59:40.808 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:40 smithi063 ceph-mon[24818]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-26T20:59:40.808 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:40 smithi063 ceph-mon[24818]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi063.front.sepia.ceph.com:9095"}]: dispatch 2024-03-26T20:59:40.808 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:40 smithi063 ceph-mon[24818]: Upgrade: Need to upgrade myself (mgr.smithi174.sexoin) 2024-03-26T20:59:40.808 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:40 smithi063 ceph-mon[24818]: Upgrade: Need to upgrade myself (mgr.smithi174.sexoin) 2024-03-26T20:59:41.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:40 smithi174 ceph-mon[26934]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-26T20:59:41.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:40 smithi174 ceph-mon[26934]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi063.front.sepia.ceph.com:9093"}]: dispatch 2024-03-26T20:59:41.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:40 smithi174 ceph-mon[26934]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-26T20:59:41.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:40 smithi174 ceph-mon[26934]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi063.front.sepia.ceph.com:3000"}]: dispatch 2024-03-26T20:59:41.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:40 smithi174 ceph-mon[26934]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-26T20:59:41.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:40 smithi174 ceph-mon[26934]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi063.front.sepia.ceph.com:9095"}]: dispatch 2024-03-26T20:59:41.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:40 smithi174 ceph-mon[26934]: Upgrade: Need to upgrade myself (mgr.smithi174.sexoin) 2024-03-26T20:59:41.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:40 smithi174 ceph-mon[26934]: Upgrade: Need to upgrade myself (mgr.smithi174.sexoin) 2024-03-26T20:59:41.472 INFO:teuthology.orchestra.run.smithi063.stdout: 5 113900 12.37 MB/sec execute 95 sec latency 42.358 ms 2024-03-26T20:59:41.760 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:41 smithi063 ceph-mon[24818]: pgmap v51: 97 pgs: 97 active+clean; 325 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.5 MiB/s wr, 575 op/s 2024-03-26T20:59:42.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:41 smithi174 ceph-mon[26934]: pgmap v51: 97 pgs: 97 active+clean; 325 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.5 MiB/s wr, 575 op/s 2024-03-26T20:59:42.472 INFO:teuthology.orchestra.run.smithi063.stdout: 5 114288 12.35 MB/sec execute 96 sec latency 38.858 ms 2024-03-26T20:59:43.320 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:43 smithi063 ceph-mon[24818]: Upgrade: Updating mgr.smithi063.lxhujt 2024-03-26T20:59:43.320 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:43 smithi063 ceph-mon[24818]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:59:43.320 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:43 smithi063 ceph-mon[24818]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi063.lxhujt", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-26T20:59:43.320 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:43 smithi063 ceph-mon[24818]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi063.lxhujt", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-26T20:59:43.320 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:43 smithi063 ceph-mon[24818]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-26T20:59:43.320 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:43 smithi063 ceph-mon[24818]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:59:43.320 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:43 smithi063 ceph-mon[24818]: Deploying daemon mgr.smithi063.lxhujt on smithi063 2024-03-26T20:59:43.320 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:43 smithi063 ceph-mon[24818]: pgmap v52: 97 pgs: 97 active+clean; 334 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 8.0 MiB/s wr, 788 op/s 2024-03-26T20:59:43.472 INFO:teuthology.orchestra.run.smithi063.stdout: 5 114646 12.30 MB/sec execute 97 sec latency 33.427 ms 2024-03-26T20:59:43.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:43 smithi174 ceph-mon[26934]: Upgrade: Updating mgr.smithi063.lxhujt 2024-03-26T20:59:43.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:43 smithi174 ceph-mon[26934]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:59:43.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:43 smithi174 ceph-mon[26934]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi063.lxhujt", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-26T20:59:43.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:43 smithi174 ceph-mon[26934]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi063.lxhujt", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-26T20:59:43.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:43 smithi174 ceph-mon[26934]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-26T20:59:43.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:43 smithi174 ceph-mon[26934]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T20:59:43.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:43 smithi174 ceph-mon[26934]: Deploying daemon mgr.smithi063.lxhujt on smithi063 2024-03-26T20:59:43.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:43 smithi174 ceph-mon[26934]: pgmap v52: 97 pgs: 97 active+clean; 334 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 8.0 MiB/s wr, 788 op/s 2024-03-26T20:59:44.472 INFO:teuthology.orchestra.run.smithi063.stdout: 5 115080 12.24 MB/sec execute 98 sec latency 71.107 ms 2024-03-26T20:59:45.472 INFO:teuthology.orchestra.run.smithi063.stdout: 5 115503 12.22 MB/sec execute 99 sec latency 38.410 ms 2024-03-26T20:59:45.844 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:45 smithi063 ceph-mon[24818]: pgmap v53: 97 pgs: 97 active+clean; 334 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.7 MiB/s wr, 457 op/s 2024-03-26T20:59:46.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:45 smithi174 ceph-mon[26934]: pgmap v53: 97 pgs: 97 active+clean; 334 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.7 MiB/s wr, 457 op/s 2024-03-26T20:59:46.472 INFO:teuthology.orchestra.run.smithi063.stdout: 5 115883 12.22 MB/sec execute 100 sec latency 51.838 ms 2024-03-26T20:59:47.472 INFO:teuthology.orchestra.run.smithi063.stdout: 5 116588 12.31 MB/sec execute 101 sec latency 35.506 ms 2024-03-26T20:59:47.865 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:47 smithi063 ceph-mon[24818]: pgmap v54: 97 pgs: 97 active+clean; 327 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.1 MiB/s wr, 707 op/s 2024-03-26T20:59:48.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:47 smithi174 ceph-mon[26934]: pgmap v54: 97 pgs: 97 active+clean; 327 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.1 MiB/s wr, 707 op/s 2024-03-26T20:59:48.473 INFO:teuthology.orchestra.run.smithi063.stdout: 5 117045 12.34 MB/sec execute 102 sec latency 63.335 ms 2024-03-26T20:59:49.473 INFO:teuthology.orchestra.run.smithi063.stdout: 5 117508 12.35 MB/sec execute 103 sec latency 24.286 ms 2024-03-26T20:59:49.884 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:49 smithi063 ceph-mon[24818]: pgmap v55: 97 pgs: 97 active+clean; 327 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.8 MiB/s wr, 463 op/s 2024-03-26T20:59:49.885 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:49 smithi063 ceph-mon[24818]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:59:49.885 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:49 smithi063 ceph-mon[24818]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:59:49.885 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:49 smithi063 ceph-mon[24818]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T20:59:49.885 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:49 smithi063 ceph-mon[24818]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T20:59:50.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:49 smithi174 ceph-mon[26934]: pgmap v55: 97 pgs: 97 active+clean; 327 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.8 MiB/s wr, 463 op/s 2024-03-26T20:59:50.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:49 smithi174 ceph-mon[26934]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:59:50.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:49 smithi174 ceph-mon[26934]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:59:50.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:49 smithi174 ceph-mon[26934]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T20:59:50.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:49 smithi174 ceph-mon[26934]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T20:59:50.472 INFO:teuthology.orchestra.run.smithi063.stdout: 5 117848 12.32 MB/sec execute 104 sec latency 54.809 ms 2024-03-26T20:59:50.915 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:50 smithi063 ceph-mon[24818]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:59:50.916 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:50 smithi063 ceph-mon[24818]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:59:50.916 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:50 smithi063 ceph-mon[24818]: pgmap v56: 97 pgs: 97 active+clean; 327 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.8 MiB/s wr, 463 op/s 2024-03-26T20:59:51.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:50 smithi174 ceph-mon[26934]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:59:51.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:50 smithi174 ceph-mon[26934]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:59:51.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:50 smithi174 ceph-mon[26934]: pgmap v56: 97 pgs: 97 active+clean; 327 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.8 MiB/s wr, 463 op/s 2024-03-26T20:59:51.473 INFO:teuthology.orchestra.run.smithi063.stdout: 5 118196 12.28 MB/sec execute 105 sec latency 46.385 ms 2024-03-26T20:59:52.473 INFO:teuthology.orchestra.run.smithi063.stdout: 5 118573 12.21 MB/sec execute 106 sec latency 49.316 ms 2024-03-26T20:59:53.473 INFO:teuthology.orchestra.run.smithi063.stdout: 5 118987 12.18 MB/sec execute 107 sec latency 44.029 ms 2024-03-26T20:59:53.490 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:53 smithi063 ceph-mon[24818]: pgmap v57: 97 pgs: 97 active+clean; 333 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 7.9 MiB/s wr, 736 op/s 2024-03-26T20:59:53.491 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:53 smithi063 ceph-mon[24818]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:59:53.491 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:53 smithi063 ceph-mon[24818]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:59:53.595 INFO:teuthology.orchestra.run.smithi063.stdout:true 2024-03-26T20:59:53.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:53 smithi174 ceph-mon[26934]: pgmap v57: 97 pgs: 97 active+clean; 333 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 7.9 MiB/s wr, 736 op/s 2024-03-26T20:59:53.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:53 smithi174 ceph-mon[26934]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:59:53.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:53 smithi174 ceph-mon[26934]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T20:59:54.473 INFO:teuthology.orchestra.run.smithi063.stdout: 5 119262 12.15 MB/sec execute 108 sec latency 50.214 ms 2024-03-26T20:59:54.527 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:54 smithi063 ceph-mon[24818]: from='client.14762 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:59:54.623 INFO:teuthology.orchestra.run.smithi063.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-26T20:59:54.623 INFO:teuthology.orchestra.run.smithi063.stdout:alertmanager.smithi063 smithi063 *:9093,9094 starting - - - - 2024-03-26T20:59:54.623 INFO:teuthology.orchestra.run.smithi063.stdout:crash.smithi063 smithi063 running (10m) 95s ago 10m 6409k - 17.2.0 e1d6a67b021e 706c62f8af89 2024-03-26T20:59:54.623 INFO:teuthology.orchestra.run.smithi063.stdout:crash.smithi174 smithi174 running (9m) 1s ago 9m 6409k - 17.2.0 e1d6a67b021e 2c6bfe6d0ca6 2024-03-26T20:59:54.623 INFO:teuthology.orchestra.run.smithi063.stdout:grafana.smithi063 smithi063 *:3000 starting - - - - 2024-03-26T20:59:54.623 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foofs.smithi063.tzaznz smithi063 running (4m) 95s ago 4m 36.2M - 17.2.0 e1d6a67b021e 10a989fc7954 2024-03-26T20:59:54.623 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foofs.smithi174.gitdvk smithi174 running (4m) 1s ago 4m 12.2M - 17.2.0 e1d6a67b021e 4339a5d98d1d 2024-03-26T20:59:54.623 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.smithi063.lxhujt smithi063 *:8443,9283,8765 starting - - - - 2024-03-26T20:59:54.623 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.smithi174.sexoin smithi174 *:8443,9283 running (2m) 1s ago 9m 507M - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 c7ed959f9018 2024-03-26T20:59:54.623 INFO:teuthology.orchestra.run.smithi063.stdout:mon.smithi063 smithi063 running (11m) 95s ago 11m 47.7M 2048M 17.2.0 e1d6a67b021e e6bd58b6c0f9 2024-03-26T20:59:54.624 INFO:teuthology.orchestra.run.smithi063.stdout:mon.smithi174 smithi174 running (9m) 1s ago 9m 42.2M 2048M 17.2.0 e1d6a67b021e 0f81c09f0c0a 2024-03-26T20:59:54.624 INFO:teuthology.orchestra.run.smithi063.stdout:nfs.foo.0.0.smithi063.fgmpwa smithi063 *:2049 running (4m) 95s ago 4m 256M - 4.0 e1d6a67b021e 3d6c0dd9ce51 2024-03-26T20:59:54.624 INFO:teuthology.orchestra.run.smithi063.stdout:nfs.foo.1.0.smithi174.pytptt smithi174 *:2049 running (3m) 1s ago 3m 81.6M - 4.0 e1d6a67b021e f220dbb449a3 2024-03-26T20:59:54.624 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.smithi063 smithi063 *:9100 starting - - - - 2024-03-26T20:59:54.624 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.smithi174 smithi174 *:9100 running (15s) 1s ago 9m 9248k - 1.3.1 1dbe0e931976 0c027f39a4c8 2024-03-26T20:59:54.624 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0 smithi063 running (8m) 95s ago 8m 294M 1936M 17.2.0 e1d6a67b021e 761b1388d8fa 2024-03-26T20:59:54.624 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1 smithi063 running (8m) 95s ago 8m 237M 1936M 17.2.0 e1d6a67b021e 108275c17cad 2024-03-26T20:59:54.624 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2 smithi063 running (7m) 95s ago 7m 176M 1936M 17.2.0 e1d6a67b021e 107f685ad08d 2024-03-26T20:59:54.624 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3 smithi063 running (7m) 95s ago 7m 263M 1936M 17.2.0 e1d6a67b021e ab0f1a855957 2024-03-26T20:59:54.624 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4 smithi174 running (6m) 1s ago 6m 396M 2704M 17.2.0 e1d6a67b021e a0e877ecc06d 2024-03-26T20:59:54.624 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5 smithi174 running (6m) 1s ago 6m 445M 2704M 17.2.0 e1d6a67b021e 378c2aca7e71 2024-03-26T20:59:54.624 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6 smithi174 running (6m) 1s ago 6m 380M 2704M 17.2.0 e1d6a67b021e ffbb5b7ad3b6 2024-03-26T20:59:54.624 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7 smithi174 running (5m) 1s ago 5m 519M 2704M 17.2.0 e1d6a67b021e c1d57fcabb3f 2024-03-26T20:59:54.624 INFO:teuthology.orchestra.run.smithi063.stdout:prometheus.smithi063 smithi063 *:9095 starting - - - - 2024-03-26T20:59:54.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:54 smithi174 ceph-mon[26934]: from='client.14762 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:59:55.314 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-03-26T20:59:55.314 INFO:teuthology.orchestra.run.smithi063.stdout: "mon": { 2024-03-26T20:59:55.314 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-26T20:59:55.314 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T20:59:55.314 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr": { 2024-03-26T20:59:55.314 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-03-26T20:59:55.314 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 1 2024-03-26T20:59:55.314 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T20:59:55.314 INFO:teuthology.orchestra.run.smithi063.stdout: "osd": { 2024-03-26T20:59:55.315 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-03-26T20:59:55.315 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T20:59:55.315 INFO:teuthology.orchestra.run.smithi063.stdout: "mds": { 2024-03-26T20:59:55.315 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-26T20:59:55.315 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T20:59:55.315 INFO:teuthology.orchestra.run.smithi063.stdout: "overall": { 2024-03-26T20:59:55.315 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-03-26T20:59:55.315 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 1 2024-03-26T20:59:55.315 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-03-26T20:59:55.315 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-03-26T20:59:55.473 INFO:teuthology.orchestra.run.smithi063.stdout: 5 119929 12.23 MB/sec execute 109 sec latency 24.676 ms 2024-03-26T20:59:55.533 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:55 smithi063 ceph-mon[24818]: from='client.24537 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:59:55.533 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:55 smithi063 ceph-mon[24818]: from='client.14770 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:59:55.533 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:55 smithi063 ceph-mon[24818]: pgmap v58: 97 pgs: 97 active+clean; 333 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.4 MiB/s wr, 522 op/s 2024-03-26T20:59:55.533 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:55 smithi063 ceph-mon[24818]: from='client.? 172.21.15.63:0/3463499938' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T20:59:55.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:55 smithi174 ceph-mon[26934]: from='client.24537 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:59:55.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:55 smithi174 ceph-mon[26934]: from='client.14770 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:59:55.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:55 smithi174 ceph-mon[26934]: pgmap v58: 97 pgs: 97 active+clean; 333 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.4 MiB/s wr, 522 op/s 2024-03-26T20:59:55.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:55 smithi174 ceph-mon[26934]: from='client.? 172.21.15.63:0/3463499938' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T20:59:55.914 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-03-26T20:59:55.914 INFO:teuthology.orchestra.run.smithi063.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d", 2024-03-26T20:59:55.914 INFO:teuthology.orchestra.run.smithi063.stdout: "in_progress": true, 2024-03-26T20:59:55.914 INFO:teuthology.orchestra.run.smithi063.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-26T20:59:55.914 INFO:teuthology.orchestra.run.smithi063.stdout: "services_complete": [], 2024-03-26T20:59:55.914 INFO:teuthology.orchestra.run.smithi063.stdout: "progress": "", 2024-03-26T20:59:55.914 INFO:teuthology.orchestra.run.smithi063.stdout: "message": "Currently upgrading mgr daemons", 2024-03-26T20:59:55.914 INFO:teuthology.orchestra.run.smithi063.stdout: "is_paused": false 2024-03-26T20:59:55.915 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-03-26T20:59:56.473 INFO:teuthology.orchestra.run.smithi063.stdout: 5 120533 12.30 MB/sec execute 110 sec latency 23.812 ms 2024-03-26T20:59:56.623 INFO:teuthology.orchestra.run.smithi063.stdout:HEALTH_OK 2024-03-26T20:59:56.788 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:56 smithi063 ceph-mon[24818]: from='client.? 172.21.15.63:0/3272200764' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-26T20:59:57.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:56 smithi174 ceph-mon[26934]: from='client.? 172.21.15.63:0/3272200764' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-26T20:59:57.473 INFO:teuthology.orchestra.run.smithi063.stdout: 5 121102 12.32 MB/sec execute 111 sec latency 31.589 ms 2024-03-26T20:59:57.796 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:57 smithi063 ceph-mon[24818]: from='client.24547 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:59:57.797 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:57 smithi063 ceph-mon[24818]: pgmap v59: 97 pgs: 97 active+clean; 329 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.9 MiB/s wr, 772 op/s 2024-03-26T20:59:58.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:57 smithi174 ceph-mon[26934]: from='client.24547 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T20:59:58.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:57 smithi174 ceph-mon[26934]: pgmap v59: 97 pgs: 97 active+clean; 329 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.9 MiB/s wr, 772 op/s 2024-03-26T20:59:58.473 INFO:teuthology.orchestra.run.smithi063.stdout: 5 121530 12.32 MB/sec execute 112 sec latency 37.487 ms 2024-03-26T20:59:59.473 INFO:teuthology.orchestra.run.smithi063.stdout: 5 121939 12.29 MB/sec execute 113 sec latency 56.418 ms 2024-03-26T20:59:59.851 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:59 smithi063 ceph-mon[24818]: pgmap v60: 97 pgs: 97 active+clean; 329 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.5 MiB/s wr, 522 op/s 2024-03-26T20:59:59.852 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:59 smithi063 ceph-mon[24818]: Standby manager daemon smithi063.lxhujt restarted 2024-03-26T20:59:59.852 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:59 smithi063 ceph-mon[24818]: Standby manager daemon smithi063.lxhujt started 2024-03-26T20:59:59.852 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:59 smithi063 ceph-mon[24818]: from='mgr.? 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi063.lxhujt/crt"}]: dispatch 2024-03-26T20:59:59.852 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:59 smithi063 ceph-mon[24818]: from='mgr.? 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-26T20:59:59.852 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:59 smithi063 ceph-mon[24818]: from='mgr.? 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi063.lxhujt/key"}]: dispatch 2024-03-26T20:59:59.852 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 20:59:59 smithi063 ceph-mon[24818]: from='mgr.? 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-26T21:00:00.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:59 smithi174 ceph-mon[26934]: pgmap v60: 97 pgs: 97 active+clean; 329 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.5 MiB/s wr, 522 op/s 2024-03-26T21:00:00.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:59 smithi174 ceph-mon[26934]: Standby manager daemon smithi063.lxhujt restarted 2024-03-26T21:00:00.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:59 smithi174 ceph-mon[26934]: Standby manager daemon smithi063.lxhujt started 2024-03-26T21:00:00.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:59 smithi174 ceph-mon[26934]: from='mgr.? 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi063.lxhujt/crt"}]: dispatch 2024-03-26T21:00:00.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:59 smithi174 ceph-mon[26934]: from='mgr.? 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-26T21:00:00.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:59 smithi174 ceph-mon[26934]: from='mgr.? 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi063.lxhujt/key"}]: dispatch 2024-03-26T21:00:00.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 20:59:59 smithi174 ceph-mon[26934]: from='mgr.? 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-26T21:00:00.473 INFO:teuthology.orchestra.run.smithi063.stdout: 5 122312 12.21 MB/sec execute 114 sec latency 39.465 ms 2024-03-26T21:00:00.885 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:00 smithi063 ceph-mon[24818]: mgrmap e28: smithi174.sexoin(active, since 115s), standbys: smithi063.lxhujt 2024-03-26T21:00:00.885 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:00 smithi063 ceph-mon[24818]: overall HEALTH_OK 2024-03-26T21:00:01.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:00 smithi174 ceph-mon[26934]: mgrmap e28: smithi174.sexoin(active, since 115s), standbys: smithi063.lxhujt 2024-03-26T21:00:01.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:00 smithi174 ceph-mon[26934]: overall HEALTH_OK 2024-03-26T21:00:01.473 INFO:teuthology.orchestra.run.smithi063.stdout: 5 122694 12.22 MB/sec execute 115 sec latency 40.624 ms 2024-03-26T21:00:01.871 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:01 smithi063 ceph-mon[24818]: pgmap v61: 97 pgs: 97 active+clean; 329 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.5 MiB/s wr, 522 op/s 2024-03-26T21:00:01.871 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:01 smithi063 ceph-mon[24818]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T21:00:01.871 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:01 smithi063 ceph-mon[24818]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T21:00:02.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:01 smithi174 ceph-mon[26934]: pgmap v61: 97 pgs: 97 active+clean; 329 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.5 MiB/s wr, 522 op/s 2024-03-26T21:00:02.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:01 smithi174 ceph-mon[26934]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T21:00:02.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:01 smithi174 ceph-mon[26934]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T21:00:02.474 INFO:teuthology.orchestra.run.smithi063.stdout: 5 123336 12.30 MB/sec execute 116 sec latency 36.687 ms 2024-03-26T21:00:03.474 INFO:teuthology.orchestra.run.smithi063.stdout: 5 123823 12.31 MB/sec execute 117 sec latency 27.346 ms 2024-03-26T21:00:03.652 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:03 smithi063 ceph-mon[24818]: pgmap v62: 97 pgs: 97 active+clean; 329 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.3 MiB/s wr, 813 op/s 2024-03-26T21:00:03.652 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:03 smithi063 ceph-mon[24818]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T21:00:03.652 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:03 smithi063 ceph-mon[24818]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T21:00:03.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:03 smithi174 ceph-mon[26934]: pgmap v62: 97 pgs: 97 active+clean; 329 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.3 MiB/s wr, 813 op/s 2024-03-26T21:00:03.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:03 smithi174 ceph-mon[26934]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T21:00:03.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:03 smithi174 ceph-mon[26934]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T21:00:04.473 INFO:teuthology.orchestra.run.smithi063.stdout: 5 124159 12.29 MB/sec execute 118 sec latency 39.262 ms 2024-03-26T21:00:05.474 INFO:teuthology.orchestra.run.smithi063.stdout: 5 124570 12.26 MB/sec execute 119 sec latency 43.309 ms 2024-03-26T21:00:05.961 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:05 smithi063 ceph-mon[24818]: pgmap v63: 97 pgs: 97 active+clean; 329 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.3 MiB/s wr, 540 op/s 2024-03-26T21:00:05.962 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:05 smithi063 ceph-mon[24818]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:00:06.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:05 smithi174 ceph-mon[26934]: pgmap v63: 97 pgs: 97 active+clean; 329 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.3 MiB/s wr, 540 op/s 2024-03-26T21:00:06.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:05 smithi174 ceph-mon[26934]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:00:06.475 INFO:teuthology.orchestra.run.smithi063.stdout: 5 124938 12.28 MB/sec execute 120 sec latency 45.419 ms 2024-03-26T21:00:07.474 INFO:teuthology.orchestra.run.smithi063.stdout: 5 125530 12.27 MB/sec execute 121 sec latency 56.694 ms 2024-03-26T21:00:07.739 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:07 smithi063 ceph-mon[24818]: pgmap v64: 97 pgs: 97 active+clean; 334 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.3 MiB/s wr, 805 op/s 2024-03-26T21:00:08.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:07 smithi174 ceph-mon[26934]: pgmap v64: 97 pgs: 97 active+clean; 334 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.3 MiB/s wr, 805 op/s 2024-03-26T21:00:08.474 INFO:teuthology.orchestra.run.smithi063.stdout: 5 126002 12.22 MB/sec execute 122 sec latency 41.844 ms 2024-03-26T21:00:09.474 INFO:teuthology.orchestra.run.smithi063.stdout: 5 126354 12.24 MB/sec execute 123 sec latency 36.749 ms 2024-03-26T21:00:09.780 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:09 smithi063 ceph-mon[24818]: pgmap v65: 97 pgs: 97 active+clean; 334 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.8 MiB/s wr, 555 op/s 2024-03-26T21:00:10.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:09 smithi174 ceph-mon[26934]: pgmap v65: 97 pgs: 97 active+clean; 334 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.8 MiB/s wr, 555 op/s 2024-03-26T21:00:10.474 INFO:teuthology.orchestra.run.smithi063.stdout: 5 126790 12.26 MB/sec execute 124 sec latency 50.014 ms 2024-03-26T21:00:11.055 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:10 smithi063 ceph-mon[24818]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T21:00:11.055 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:10 smithi063 ceph-mon[24818]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T21:00:11.055 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:10 smithi063 ceph-mon[24818]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:00:11.055 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:10 smithi063 ceph-mon[24818]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T21:00:11.055 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:10 smithi063 ceph-mon[24818]: pgmap v66: 97 pgs: 97 active+clean; 334 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.2 MiB/s wr, 589 op/s 2024-03-26T21:00:11.055 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:10 smithi063 ceph-mon[24818]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T21:00:11.056 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:10 smithi063 ceph-mon[24818]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T21:00:11.056 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:10 smithi063 ceph-mon[24818]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:00:11.056 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:10 smithi063 ceph-mon[24818]: Upgrade: Need to upgrade myself (mgr.smithi174.sexoin) 2024-03-26T21:00:11.056 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:10 smithi063 ceph-mon[24818]: Upgrade: Need to upgrade myself (mgr.smithi174.sexoin) 2024-03-26T21:00:11.056 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:10 smithi063 ceph-mon[24818]: Failing over to other MGR 2024-03-26T21:00:11.056 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:10 smithi063 ceph-mon[24818]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "mgr fail", "who": "smithi174.sexoin"}]: dispatch 2024-03-26T21:00:11.056 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:10 smithi063 ceph-mon[24818]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "mgr fail", "who": "smithi174.sexoin"}]: dispatch 2024-03-26T21:00:11.056 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:10 smithi063 ceph-mon[24818]: osdmap e70: 8 total, 8 up, 8 in 2024-03-26T21:00:11.056 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:10 smithi063 ceph-mon[24818]: Standby manager daemon smithi174.sexoin started 2024-03-26T21:00:11.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:10 smithi174 ceph-mon[26934]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T21:00:11.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:10 smithi174 ceph-mon[26934]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T21:00:11.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:10 smithi174 ceph-mon[26934]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:00:11.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:10 smithi174 ceph-mon[26934]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T21:00:11.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:10 smithi174 ceph-mon[26934]: pgmap v66: 97 pgs: 97 active+clean; 334 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.2 MiB/s wr, 589 op/s 2024-03-26T21:00:11.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:10 smithi174 ceph-mon[26934]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T21:00:11.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:10 smithi174 ceph-mon[26934]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' 2024-03-26T21:00:11.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:10 smithi174 ceph-mon[26934]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:00:11.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:10 smithi174 ceph-mon[26934]: Upgrade: Need to upgrade myself (mgr.smithi174.sexoin) 2024-03-26T21:00:11.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:10 smithi174 ceph-mon[26934]: Upgrade: Need to upgrade myself (mgr.smithi174.sexoin) 2024-03-26T21:00:11.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:10 smithi174 ceph-mon[26934]: Failing over to other MGR 2024-03-26T21:00:11.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:10 smithi174 ceph-mon[26934]: from='mgr.24449 172.21.15.174:0/2577307008' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "mgr fail", "who": "smithi174.sexoin"}]: dispatch 2024-03-26T21:00:11.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:10 smithi174 ceph-mon[26934]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "mgr fail", "who": "smithi174.sexoin"}]: dispatch 2024-03-26T21:00:11.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:10 smithi174 ceph-mon[26934]: osdmap e70: 8 total, 8 up, 8 in 2024-03-26T21:00:11.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:10 smithi174 ceph-mon[26934]: Standby manager daemon smithi174.sexoin started 2024-03-26T21:00:11.474 INFO:teuthology.orchestra.run.smithi063.stdout: 5 127187 12.24 MB/sec execute 125 sec latency 45.787 ms 2024-03-26T21:00:12.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:11 smithi174 ceph-mon[26934]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-03-26T21:00:12.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:11 smithi174 ceph-mon[26934]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' cmd='[{"prefix": "mgr fail", "who": "smithi174.sexoin"}]': finished 2024-03-26T21:00:12.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:11 smithi174 ceph-mon[26934]: mgrmap e29: smithi063.lxhujt(active, starting, since 0.86224s), standbys: smithi174.sexoin 2024-03-26T21:00:12.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:11 smithi174 ceph-mon[26934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mon metadata", "id": "smithi063"}]: dispatch 2024-03-26T21:00:12.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:11 smithi174 ceph-mon[26934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mon metadata", "id": "smithi174"}]: dispatch 2024-03-26T21:00:12.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:11 smithi174 ceph-mon[26934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi063.tzaznz"}]: dispatch 2024-03-26T21:00:12.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:11 smithi174 ceph-mon[26934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi174.gitdvk"}]: dispatch 2024-03-26T21:00:12.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:11 smithi174 ceph-mon[26934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mgr metadata", "who": "smithi063.lxhujt", "id": "smithi063.lxhujt"}]: dispatch 2024-03-26T21:00:12.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:11 smithi174 ceph-mon[26934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mgr metadata", "who": "smithi174.sexoin", "id": "smithi174.sexoin"}]: dispatch 2024-03-26T21:00:12.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:11 smithi174 ceph-mon[26934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-26T21:00:12.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:11 smithi174 ceph-mon[26934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-26T21:00:12.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:11 smithi174 ceph-mon[26934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-26T21:00:12.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:11 smithi174 ceph-mon[26934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-26T21:00:12.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:11 smithi174 ceph-mon[26934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-26T21:00:12.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:11 smithi174 ceph-mon[26934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-26T21:00:12.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:11 smithi174 ceph-mon[26934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-26T21:00:12.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:11 smithi174 ceph-mon[26934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-26T21:00:12.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:11 smithi174 ceph-mon[26934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-26T21:00:12.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:11 smithi174 ceph-mon[26934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-26T21:00:12.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:11 smithi174 ceph-mon[26934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-26T21:00:12.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:11 smithi174 ceph-mon[26934]: Manager daemon smithi063.lxhujt is now available 2024-03-26T21:00:12.050 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:11 smithi174 ceph-mon[26934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:00:12.050 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:11 smithi174 ceph-mon[26934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' 2024-03-26T21:00:12.050 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:11 smithi174 ceph-mon[26934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:00:12.050 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:11 smithi174 ceph-mon[26934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi063.lxhujt/mirror_snapshot_schedule"}]: dispatch 2024-03-26T21:00:12.050 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:11 smithi174 ceph-mon[26934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi063.lxhujt/trash_purge_schedule"}]: dispatch 2024-03-26T21:00:12.077 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:11 smithi063 ceph-mon[24818]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-03-26T21:00:12.077 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:11 smithi063 ceph-mon[24818]: from='mgr.24449 ' entity='mgr.smithi174.sexoin' cmd='[{"prefix": "mgr fail", "who": "smithi174.sexoin"}]': finished 2024-03-26T21:00:12.077 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:11 smithi063 ceph-mon[24818]: mgrmap e29: smithi063.lxhujt(active, starting, since 0.86224s), standbys: smithi174.sexoin 2024-03-26T21:00:12.077 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:11 smithi063 ceph-mon[24818]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mon metadata", "id": "smithi063"}]: dispatch 2024-03-26T21:00:12.077 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:11 smithi063 ceph-mon[24818]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mon metadata", "id": "smithi174"}]: dispatch 2024-03-26T21:00:12.077 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:11 smithi063 ceph-mon[24818]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi063.tzaznz"}]: dispatch 2024-03-26T21:00:12.077 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:11 smithi063 ceph-mon[24818]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi174.gitdvk"}]: dispatch 2024-03-26T21:00:12.077 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:11 smithi063 ceph-mon[24818]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mgr metadata", "who": "smithi063.lxhujt", "id": "smithi063.lxhujt"}]: dispatch 2024-03-26T21:00:12.077 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:11 smithi063 ceph-mon[24818]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mgr metadata", "who": "smithi174.sexoin", "id": "smithi174.sexoin"}]: dispatch 2024-03-26T21:00:12.077 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:11 smithi063 ceph-mon[24818]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-26T21:00:12.078 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:11 smithi063 ceph-mon[24818]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-26T21:00:12.078 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:11 smithi063 ceph-mon[24818]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-26T21:00:12.078 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:11 smithi063 ceph-mon[24818]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-26T21:00:12.078 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:11 smithi063 ceph-mon[24818]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-26T21:00:12.078 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:11 smithi063 ceph-mon[24818]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-26T21:00:12.078 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:11 smithi063 ceph-mon[24818]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-26T21:00:12.078 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:11 smithi063 ceph-mon[24818]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-26T21:00:12.078 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:11 smithi063 ceph-mon[24818]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-26T21:00:12.078 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:11 smithi063 ceph-mon[24818]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-26T21:00:12.078 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:11 smithi063 ceph-mon[24818]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-26T21:00:12.078 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:11 smithi063 ceph-mon[24818]: Manager daemon smithi063.lxhujt is now available 2024-03-26T21:00:12.078 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:11 smithi063 ceph-mon[24818]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:00:12.078 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:11 smithi063 ceph-mon[24818]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' 2024-03-26T21:00:12.078 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:11 smithi063 ceph-mon[24818]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:00:12.078 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:11 smithi063 ceph-mon[24818]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi063.lxhujt/mirror_snapshot_schedule"}]: dispatch 2024-03-26T21:00:12.079 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:11 smithi063 ceph-mon[24818]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi063.lxhujt/trash_purge_schedule"}]: dispatch 2024-03-26T21:00:12.474 INFO:teuthology.orchestra.run.smithi063.stdout: 5 127419 12.20 MB/sec execute 126 sec latency 47.279 ms 2024-03-26T21:00:13.113 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:13 smithi063 ceph-mon[24818]: [26/Mar/2024:21:00:11] ENGINE Bus STARTING 2024-03-26T21:00:13.113 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:13 smithi063 ceph-mon[24818]: mgrmap e30: smithi063.lxhujt(active, since 1.86649s), standbys: smithi174.sexoin 2024-03-26T21:00:13.113 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:13 smithi063 ceph-mon[24818]: pgmap v3: 97 pgs: 97 active+clean; 331 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-03-26T21:00:13.114 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:13 smithi063 ceph-mon[24818]: [26/Mar/2024:21:00:12] ENGINE Serving on http://172.21.15.63:8765 2024-03-26T21:00:13.114 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:13 smithi063 ceph-mon[24818]: [26/Mar/2024:21:00:12] ENGINE Serving on https://172.21.15.63:7150 2024-03-26T21:00:13.114 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:13 smithi063 ceph-mon[24818]: [26/Mar/2024:21:00:12] ENGINE Bus STARTED 2024-03-26T21:00:13.114 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:13 smithi063 ceph-mon[24818]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' 2024-03-26T21:00:13.114 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:13 smithi063 ceph-mon[24818]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' 2024-03-26T21:00:13.257 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:13 smithi174 ceph-mon[26934]: [26/Mar/2024:21:00:11] ENGINE Bus STARTING 2024-03-26T21:00:13.257 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:13 smithi174 ceph-mon[26934]: mgrmap e30: smithi063.lxhujt(active, since 1.86649s), standbys: smithi174.sexoin 2024-03-26T21:00:13.257 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:13 smithi174 ceph-mon[26934]: pgmap v3: 97 pgs: 97 active+clean; 331 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-03-26T21:00:13.257 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:13 smithi174 ceph-mon[26934]: [26/Mar/2024:21:00:12] ENGINE Serving on http://172.21.15.63:8765 2024-03-26T21:00:13.257 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:13 smithi174 ceph-mon[26934]: [26/Mar/2024:21:00:12] ENGINE Serving on https://172.21.15.63:7150 2024-03-26T21:00:13.258 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:13 smithi174 ceph-mon[26934]: [26/Mar/2024:21:00:12] ENGINE Bus STARTED 2024-03-26T21:00:13.258 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:13 smithi174 ceph-mon[26934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' 2024-03-26T21:00:13.258 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:13 smithi174 ceph-mon[26934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' 2024-03-26T21:00:13.474 INFO:teuthology.orchestra.run.smithi063.stdout: 5 127682 12.16 MB/sec execute 127 sec latency 47.804 ms 2024-03-26T21:00:14.131 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:14 smithi063 ceph-mon[24818]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-03-26T21:00:14.132 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:14 smithi063 ceph-mon[24818]: Cluster is now healthy 2024-03-26T21:00:14.132 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:14 smithi063 ceph-mon[24818]: pgmap v4: 97 pgs: 97 active+clean; 331 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-03-26T21:00:14.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:14 smithi174 ceph-mon[26934]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-03-26T21:00:14.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:14 smithi174 ceph-mon[26934]: Cluster is now healthy 2024-03-26T21:00:14.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:14 smithi174 ceph-mon[26934]: pgmap v4: 97 pgs: 97 active+clean; 331 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-03-26T21:00:14.474 INFO:teuthology.orchestra.run.smithi063.stdout: 5 127977 12.13 MB/sec execute 128 sec latency 52.601 ms 2024-03-26T21:00:15.145 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:15 smithi063 ceph-mon[24818]: mgrmap e31: smithi063.lxhujt(active, since 3s), standbys: smithi174.sexoin 2024-03-26T21:00:15.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:15 smithi174 ceph-mon[26934]: mgrmap e31: smithi063.lxhujt(active, since 3s), standbys: smithi174.sexoin 2024-03-26T21:00:15.474 INFO:teuthology.orchestra.run.smithi063.stdout: 5 128251 12.09 MB/sec execute 129 sec latency 60.319 ms 2024-03-26T21:00:16.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:16 smithi174 ceph-mon[26934]: pgmap v5: 97 pgs: 97 active+clean; 331 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-03-26T21:00:16.179 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:16 smithi063 ceph-mon[24818]: pgmap v5: 97 pgs: 97 active+clean; 331 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-03-26T21:00:16.474 INFO:teuthology.orchestra.run.smithi063.stdout: 5 128510 12.07 MB/sec execute 130 sec latency 45.747 ms 2024-03-26T21:00:17.465 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:17 smithi063 ceph-mon[24818]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' 2024-03-26T21:00:17.465 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:17 smithi063 ceph-mon[24818]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' 2024-03-26T21:00:17.474 INFO:teuthology.orchestra.run.smithi063.stdout: 5 128769 12.02 MB/sec execute 131 sec latency 75.786 ms 2024-03-26T21:00:17.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:17 smithi174 ceph-mon[26934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' 2024-03-26T21:00:17.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:17 smithi174 ceph-mon[26934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' 2024-03-26T21:00:18.474 INFO:teuthology.orchestra.run.smithi063.stdout: 5 129093 11.97 MB/sec execute 132 sec latency 74.893 ms 2024-03-26T21:00:18.978 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:18 smithi063 ceph-mon[24818]: pgmap v6: 97 pgs: 97 active+clean; 335 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 1.9 MiB/s wr, 238 op/s 2024-03-26T21:00:18.979 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:18 smithi063 ceph-mon[24818]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' 2024-03-26T21:00:18.979 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:18 smithi063 ceph-mon[24818]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' 2024-03-26T21:00:19.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:18 smithi174 ceph-mon[26934]: pgmap v6: 97 pgs: 97 active+clean; 335 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 1.9 MiB/s wr, 238 op/s 2024-03-26T21:00:19.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:18 smithi174 ceph-mon[26934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' 2024-03-26T21:00:19.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:18 smithi174 ceph-mon[26934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' 2024-03-26T21:00:19.474 INFO:teuthology.orchestra.run.smithi063.stdout: 5 129455 11.92 MB/sec execute 133 sec latency 72.945 ms 2024-03-26T21:00:20.476 INFO:teuthology.orchestra.run.smithi063.stdout: 5 129934 11.96 MB/sec execute 134 sec latency 43.116 ms 2024-03-26T21:00:21.004 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:20 smithi063 ceph-mon[24818]: pgmap v7: 97 pgs: 97 active+clean; 335 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 35 KiB/s rd, 3.3 MiB/s wr, 321 op/s 2024-03-26T21:00:21.004 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:20 smithi063 ceph-mon[24818]: from='mgr.? 172.21.15.174:0/1399484501' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi174.sexoin/crt"}]: dispatch 2024-03-26T21:00:21.004 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:20 smithi063 ceph-mon[24818]: from='mgr.? 172.21.15.174:0/1399484501' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-26T21:00:21.004 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:20 smithi063 ceph-mon[24818]: from='mgr.? 172.21.15.174:0/1399484501' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi174.sexoin/key"}]: dispatch 2024-03-26T21:00:21.004 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:20 smithi063 ceph-mon[24818]: from='mgr.? 172.21.15.174:0/1399484501' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-26T21:00:21.004 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:20 smithi063 ceph-mon[24818]: Standby manager daemon smithi174.sexoin restarted 2024-03-26T21:00:21.004 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:20 smithi063 ceph-mon[24818]: Standby manager daemon smithi174.sexoin started 2024-03-26T21:00:21.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:20 smithi174 ceph-mon[26934]: pgmap v7: 97 pgs: 97 active+clean; 335 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 35 KiB/s rd, 3.3 MiB/s wr, 321 op/s 2024-03-26T21:00:21.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:20 smithi174 ceph-mon[26934]: from='mgr.? 172.21.15.174:0/1399484501' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi174.sexoin/crt"}]: dispatch 2024-03-26T21:00:21.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:20 smithi174 ceph-mon[26934]: from='mgr.? 172.21.15.174:0/1399484501' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-26T21:00:21.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:20 smithi174 ceph-mon[26934]: from='mgr.? 172.21.15.174:0/1399484501' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi174.sexoin/key"}]: dispatch 2024-03-26T21:00:21.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:20 smithi174 ceph-mon[26934]: from='mgr.? 172.21.15.174:0/1399484501' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-26T21:00:21.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:20 smithi174 ceph-mon[26934]: Standby manager daemon smithi174.sexoin restarted 2024-03-26T21:00:21.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:20 smithi174 ceph-mon[26934]: Standby manager daemon smithi174.sexoin started 2024-03-26T21:00:21.476 INFO:teuthology.orchestra.run.smithi063.stdout: 5 130442 11.99 MB/sec execute 135 sec latency 29.063 ms 2024-03-26T21:00:22.029 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:21 smithi063 ceph-mon[24818]: mgrmap e32: smithi063.lxhujt(active, since 10s), standbys: smithi174.sexoin 2024-03-26T21:00:22.029 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:21 smithi063 ceph-mon[24818]: pgmap v8: 97 pgs: 97 active+clean; 335 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 27 KiB/s rd, 2.6 MiB/s wr, 249 op/s 2024-03-26T21:00:22.029 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:21 smithi063 ceph-mon[24818]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' 2024-03-26T21:00:22.029 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:21 smithi063 ceph-mon[24818]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' 2024-03-26T21:00:22.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:21 smithi174 ceph-mon[26934]: mgrmap e32: smithi063.lxhujt(active, since 10s), standbys: smithi174.sexoin 2024-03-26T21:00:22.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:21 smithi174 ceph-mon[26934]: pgmap v8: 97 pgs: 97 active+clean; 335 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 27 KiB/s rd, 2.6 MiB/s wr, 249 op/s 2024-03-26T21:00:22.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:21 smithi174 ceph-mon[26934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' 2024-03-26T21:00:22.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:21 smithi174 ceph-mon[26934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' 2024-03-26T21:00:22.476 INFO:teuthology.orchestra.run.smithi063.stdout: 5 130848 11.98 MB/sec execute 136 sec latency 40.140 ms 2024-03-26T21:00:23.476 INFO:teuthology.orchestra.run.smithi063.stdout: 5 131089 11.93 MB/sec execute 137 sec latency 51.216 ms 2024-03-26T21:00:23.840 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:23 smithi063 ceph-mon[24818]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' 2024-03-26T21:00:23.841 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:23 smithi063 ceph-mon[24818]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' 2024-03-26T21:00:24.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:23 smithi174 ceph-mon[26934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' 2024-03-26T21:00:24.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:23 smithi174 ceph-mon[26934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' 2024-03-26T21:00:24.476 INFO:teuthology.orchestra.run.smithi063.stdout: 5 131428 11.92 MB/sec execute 138 sec latency 74.559 ms 2024-03-26T21:00:24.872 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:24 smithi063 ceph-mon[24818]: pgmap v9: 97 pgs: 97 active+clean; 335 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 25 KiB/s rd, 4.5 MiB/s wr, 457 op/s 2024-03-26T21:00:25.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:24 smithi174 ceph-mon[26934]: pgmap v9: 97 pgs: 97 active+clean; 335 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 25 KiB/s rd, 4.5 MiB/s wr, 457 op/s 2024-03-26T21:00:25.476 INFO:teuthology.orchestra.run.smithi063.stdout: 5 131734 11.90 MB/sec execute 139 sec latency 51.665 ms 2024-03-26T21:00:26.476 INFO:teuthology.orchestra.run.smithi063.stdout: 5 132061 11.89 MB/sec execute 140 sec latency 38.475 ms 2024-03-26T21:00:26.905 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:26 smithi063 ceph-mon[24818]: pgmap v10: 97 pgs: 97 active+clean; 335 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 23 KiB/s rd, 4.2 MiB/s wr, 418 op/s 2024-03-26T21:00:26.906 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:26 smithi063 ceph-mon[24818]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' 2024-03-26T21:00:26.906 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:26 smithi063 ceph-mon[24818]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:00:27.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:26 smithi174 ceph-mon[26934]: pgmap v10: 97 pgs: 97 active+clean; 335 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 23 KiB/s rd, 4.2 MiB/s wr, 418 op/s 2024-03-26T21:00:27.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:26 smithi174 ceph-mon[26934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' 2024-03-26T21:00:27.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:26 smithi174 ceph-mon[26934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:00:27.156 INFO:teuthology.orchestra.run.smithi063.stdout:true 2024-03-26T21:00:27.476 INFO:teuthology.orchestra.run.smithi063.stdout: 5 132379 11.85 MB/sec execute 141 sec latency 84.469 ms 2024-03-26T21:00:28.128 INFO:teuthology.orchestra.run.smithi063.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-26T21:00:28.128 INFO:teuthology.orchestra.run.smithi063.stdout:alertmanager.smithi063 smithi063 *:9093,9094 running (102s) 6s ago 10m 17.6M - 0.23.0 ba2b418f427c 6e160c94aac3 2024-03-26T21:00:28.128 INFO:teuthology.orchestra.run.smithi063.stdout:crash.smithi063 smithi063 running (10m) 6s ago 10m 6409k - 17.2.0 e1d6a67b021e 706c62f8af89 2024-03-26T21:00:28.128 INFO:teuthology.orchestra.run.smithi063.stdout:crash.smithi174 smithi174 running (10m) 11s ago 10m 6409k - 17.2.0 e1d6a67b021e 2c6bfe6d0ca6 2024-03-26T21:00:28.128 INFO:teuthology.orchestra.run.smithi063.stdout:grafana.smithi063 smithi063 *:3000 unknown 6s ago 10m - - 2024-03-26T21:00:28.128 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foofs.smithi063.tzaznz smithi063 running (4m) 6s ago 4m 37.6M - 17.2.0 e1d6a67b021e 10a989fc7954 2024-03-26T21:00:28.128 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foofs.smithi174.gitdvk smithi174 running (4m) 11s ago 4m 12.5M - 17.2.0 e1d6a67b021e 4339a5d98d1d 2024-03-26T21:00:28.128 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.smithi063.lxhujt smithi063 *:8443,9283,8765 running (39s) 6s ago 12m 482M - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 6d9d10330a90 2024-03-26T21:00:28.128 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.smithi174.sexoin smithi174 *:8443,9283 running (2m) 11s ago 10m 105M - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 c7ed959f9018 2024-03-26T21:00:28.128 INFO:teuthology.orchestra.run.smithi063.stdout:mon.smithi063 smithi063 running (12m) 6s ago 12m 55.6M 2048M 17.2.0 e1d6a67b021e e6bd58b6c0f9 2024-03-26T21:00:28.128 INFO:teuthology.orchestra.run.smithi063.stdout:mon.smithi174 smithi174 running (10m) 11s ago 10m 45.2M 2048M 17.2.0 e1d6a67b021e 0f81c09f0c0a 2024-03-26T21:00:28.129 INFO:teuthology.orchestra.run.smithi063.stdout:nfs.foo.0.0.smithi063.fgmpwa smithi063 *:2049 running (4m) 6s ago 4m 267M - 4.0 e1d6a67b021e 3d6c0dd9ce51 2024-03-26T21:00:28.129 INFO:teuthology.orchestra.run.smithi063.stdout:nfs.foo.1.0.smithi174.pytptt smithi174 *:2049 running (4m) 11s ago 4m 81.8M - 4.0 e1d6a67b021e f220dbb449a3 2024-03-26T21:00:28.129 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.smithi063 smithi063 *:9100 running (75s) 6s ago 10m 12.6M - 1.3.1 1dbe0e931976 f52f41fd14c0 2024-03-26T21:00:28.129 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.smithi174 smithi174 *:9100 running (49s) 11s ago 9m 11.2M - 1.3.1 1dbe0e931976 0c027f39a4c8 2024-03-26T21:00:28.129 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0 smithi063 running (9m) 6s ago 9m 459M 1936M 17.2.0 e1d6a67b021e 761b1388d8fa 2024-03-26T21:00:28.129 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1 smithi063 running (8m) 6s ago 8m 405M 1936M 17.2.0 e1d6a67b021e 108275c17cad 2024-03-26T21:00:28.129 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2 smithi063 running (8m) 6s ago 8m 269M 1936M 17.2.0 e1d6a67b021e 107f685ad08d 2024-03-26T21:00:28.129 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3 smithi063 running (7m) 6s ago 7m 426M 1936M 17.2.0 e1d6a67b021e ab0f1a855957 2024-03-26T21:00:28.129 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4 smithi174 running (7m) 11s ago 7m 440M 2704M 17.2.0 e1d6a67b021e a0e877ecc06d 2024-03-26T21:00:28.129 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5 smithi174 running (7m) 11s ago 7m 483M 2704M 17.2.0 e1d6a67b021e 378c2aca7e71 2024-03-26T21:00:28.129 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6 smithi174 running (6m) 11s ago 6m 415M 2704M 17.2.0 e1d6a67b021e ffbb5b7ad3b6 2024-03-26T21:00:28.129 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7 smithi174 running (6m) 11s ago 6m 565M 2704M 17.2.0 e1d6a67b021e c1d57fcabb3f 2024-03-26T21:00:28.129 INFO:teuthology.orchestra.run.smithi063.stdout:prometheus.smithi063 smithi063 *:9095 running (52s) 6s ago 9m 55.0M - 2.33.4 514e6a882f6e fa59d46139c8 2024-03-26T21:00:28.476 INFO:teuthology.orchestra.run.smithi063.stdout: 5 132752 11.81 MB/sec execute 142 sec latency 47.707 ms 2024-03-26T21:00:28.853 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-03-26T21:00:28.854 INFO:teuthology.orchestra.run.smithi063.stdout: "mon": { 2024-03-26T21:00:28.854 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-26T21:00:28.854 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:00:28.854 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr": { 2024-03-26T21:00:28.854 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 2 2024-03-26T21:00:28.854 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:00:28.854 INFO:teuthology.orchestra.run.smithi063.stdout: "osd": { 2024-03-26T21:00:28.854 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-03-26T21:00:28.854 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:00:28.854 INFO:teuthology.orchestra.run.smithi063.stdout: "mds": { 2024-03-26T21:00:28.854 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-26T21:00:28.854 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:00:28.854 INFO:teuthology.orchestra.run.smithi063.stdout: "overall": { 2024-03-26T21:00:28.854 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-03-26T21:00:28.855 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 2 2024-03-26T21:00:28.855 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-03-26T21:00:28.855 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-03-26T21:00:28.961 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:28 smithi063 ceph-mon[24818]: pgmap v11: 97 pgs: 97 active+clean; 340 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 27 KiB/s rd, 4.9 MiB/s wr, 524 op/s 2024-03-26T21:00:28.961 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:28 smithi063 ceph-mon[24818]: from='client.14812 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:00:28.961 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:28 smithi063 ceph-mon[24818]: from='client.24581 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:00:28.961 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:28 smithi063 ceph-mon[24818]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' 2024-03-26T21:00:28.961 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:28 smithi063 ceph-mon[24818]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' 2024-03-26T21:00:28.961 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:28 smithi063 ceph-mon[24818]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:00:28.961 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:28 smithi063 ceph-mon[24818]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T21:00:29.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:28 smithi174 ceph-mon[26934]: pgmap v11: 97 pgs: 97 active+clean; 340 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 27 KiB/s rd, 4.9 MiB/s wr, 524 op/s 2024-03-26T21:00:29.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:28 smithi174 ceph-mon[26934]: from='client.14812 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:00:29.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:28 smithi174 ceph-mon[26934]: from='client.24581 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:00:29.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:28 smithi174 ceph-mon[26934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' 2024-03-26T21:00:29.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:28 smithi174 ceph-mon[26934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' 2024-03-26T21:00:29.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:28 smithi174 ceph-mon[26934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:00:29.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:28 smithi174 ceph-mon[26934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T21:00:29.415 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-03-26T21:00:29.415 INFO:teuthology.orchestra.run.smithi063.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d", 2024-03-26T21:00:29.415 INFO:teuthology.orchestra.run.smithi063.stdout: "in_progress": true, 2024-03-26T21:00:29.415 INFO:teuthology.orchestra.run.smithi063.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-26T21:00:29.415 INFO:teuthology.orchestra.run.smithi063.stdout: "services_complete": [ 2024-03-26T21:00:29.415 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr" 2024-03-26T21:00:29.415 INFO:teuthology.orchestra.run.smithi063.stdout: ], 2024-03-26T21:00:29.415 INFO:teuthology.orchestra.run.smithi063.stdout: "progress": "2/23 daemons upgraded", 2024-03-26T21:00:29.415 INFO:teuthology.orchestra.run.smithi063.stdout: "message": "", 2024-03-26T21:00:29.415 INFO:teuthology.orchestra.run.smithi063.stdout: "is_paused": false 2024-03-26T21:00:29.416 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-03-26T21:00:29.476 INFO:teuthology.orchestra.run.smithi063.stdout: 5 133129 11.78 MB/sec execute 143 sec latency 58.135 ms 2024-03-26T21:00:29.971 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:29 smithi063 ceph-mon[24818]: from='client.14820 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:00:29.971 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:29 smithi063 ceph-mon[24818]: Updating smithi063:/etc/ceph/ceph.conf 2024-03-26T21:00:29.971 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:29 smithi063 ceph-mon[24818]: Updating smithi174:/etc/ceph/ceph.conf 2024-03-26T21:00:29.971 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:29 smithi063 ceph-mon[24818]: from='client.? 172.21.15.63:0/469629707' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:00:30.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:29 smithi174 ceph-mon[26934]: from='client.14820 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:00:30.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:29 smithi174 ceph-mon[26934]: Updating smithi063:/etc/ceph/ceph.conf 2024-03-26T21:00:30.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:29 smithi174 ceph-mon[26934]: Updating smithi174:/etc/ceph/ceph.conf 2024-03-26T21:00:30.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:29 smithi174 ceph-mon[26934]: from='client.? 172.21.15.63:0/469629707' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:00:30.117 INFO:teuthology.orchestra.run.smithi063.stdout:HEALTH_OK 2024-03-26T21:00:30.476 INFO:teuthology.orchestra.run.smithi063.stdout: 5 133561 11.81 MB/sec execute 144 sec latency 51.813 ms 2024-03-26T21:00:30.981 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:30 smithi063 ceph-mon[24818]: Updating smithi063:/var/lib/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-26T21:00:30.981 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:30 smithi063 ceph-mon[24818]: Updating smithi174:/var/lib/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-26T21:00:30.981 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:30 smithi063 ceph-mon[24818]: pgmap v12: 97 pgs: 97 active+clean; 331 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 23 KiB/s rd, 5.5 MiB/s wr, 520 op/s 2024-03-26T21:00:30.981 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:30 smithi063 ceph-mon[24818]: Updating smithi063:/etc/ceph/ceph.client.admin.keyring 2024-03-26T21:00:30.981 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:30 smithi063 ceph-mon[24818]: Updating smithi174:/etc/ceph/ceph.client.admin.keyring 2024-03-26T21:00:30.981 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:30 smithi063 ceph-mon[24818]: from='client.24591 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:00:30.981 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:30 smithi063 ceph-mon[24818]: Updating smithi174:/var/lib/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-26T21:00:30.981 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:30 smithi063 ceph-mon[24818]: Updating smithi063:/var/lib/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-26T21:00:30.982 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:30 smithi063 ceph-mon[24818]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' 2024-03-26T21:00:30.982 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:30 smithi063 ceph-mon[24818]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' 2024-03-26T21:00:30.982 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:30 smithi063 ceph-mon[24818]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' 2024-03-26T21:00:30.982 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:30 smithi063 ceph-mon[24818]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' 2024-03-26T21:00:30.982 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:30 smithi063 ceph-mon[24818]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' 2024-03-26T21:00:30.982 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:30 smithi063 ceph-mon[24818]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' 2024-03-26T21:00:30.982 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:30 smithi063 ceph-mon[24818]: from='client.? 172.21.15.63:0/3237282505' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-26T21:00:31.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:30 smithi174 ceph-mon[26934]: Updating smithi063:/var/lib/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-26T21:00:31.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:30 smithi174 ceph-mon[26934]: Updating smithi174:/var/lib/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-26T21:00:31.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:30 smithi174 ceph-mon[26934]: pgmap v12: 97 pgs: 97 active+clean; 331 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 23 KiB/s rd, 5.5 MiB/s wr, 520 op/s 2024-03-26T21:00:31.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:30 smithi174 ceph-mon[26934]: Updating smithi063:/etc/ceph/ceph.client.admin.keyring 2024-03-26T21:00:31.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:30 smithi174 ceph-mon[26934]: Updating smithi174:/etc/ceph/ceph.client.admin.keyring 2024-03-26T21:00:31.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:30 smithi174 ceph-mon[26934]: from='client.24591 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:00:31.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:30 smithi174 ceph-mon[26934]: Updating smithi174:/var/lib/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-26T21:00:31.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:30 smithi174 ceph-mon[26934]: Updating smithi063:/var/lib/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-26T21:00:31.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:30 smithi174 ceph-mon[26934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' 2024-03-26T21:00:31.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:30 smithi174 ceph-mon[26934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' 2024-03-26T21:00:31.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:30 smithi174 ceph-mon[26934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' 2024-03-26T21:00:31.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:30 smithi174 ceph-mon[26934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' 2024-03-26T21:00:31.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:30 smithi174 ceph-mon[26934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' 2024-03-26T21:00:31.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:30 smithi174 ceph-mon[26934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' 2024-03-26T21:00:31.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:30 smithi174 ceph-mon[26934]: from='client.? 172.21.15.63:0/3237282505' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-26T21:00:31.476 INFO:teuthology.orchestra.run.smithi063.stdout: 5 134061 11.83 MB/sec execute 145 sec latency 68.536 ms 2024-03-26T21:00:31.756 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:31 smithi063 ceph-mon[24818]: pgmap v13: 97 pgs: 97 active+clean; 331 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.8 MiB/s wr, 476 op/s 2024-03-26T21:00:31.757 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:31 smithi063 ceph-mon[24818]: Reconfiguring prometheus.smithi063 (dependencies changed)... 2024-03-26T21:00:31.757 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:31 smithi063 ceph-mon[24818]: Reconfiguring daemon prometheus.smithi063 on smithi063 2024-03-26T21:00:31.757 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:31 smithi063 ceph-mon[24818]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-03-26T21:00:32.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:31 smithi174 ceph-mon[26934]: pgmap v13: 97 pgs: 97 active+clean; 331 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.8 MiB/s wr, 476 op/s 2024-03-26T21:00:32.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:31 smithi174 ceph-mon[26934]: Reconfiguring prometheus.smithi063 (dependencies changed)... 2024-03-26T21:00:32.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:31 smithi174 ceph-mon[26934]: Reconfiguring daemon prometheus.smithi063 on smithi063 2024-03-26T21:00:32.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:31 smithi174 ceph-mon[26934]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-03-26T21:00:32.476 INFO:teuthology.orchestra.run.smithi063.stdout: 5 134396 11.82 MB/sec execute 146 sec latency 44.943 ms 2024-03-26T21:00:33.476 INFO:teuthology.orchestra.run.smithi063.stdout: 5 134611 11.77 MB/sec execute 147 sec latency 74.672 ms 2024-03-26T21:00:33.933 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:33 smithi063 ceph-mon[24818]: pgmap v14: 97 pgs: 97 active+clean; 331 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.8 MiB/s wr, 476 op/s 2024-03-26T21:00:34.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:33 smithi174 ceph-mon[26934]: pgmap v14: 97 pgs: 97 active+clean; 331 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.8 MiB/s wr, 476 op/s 2024-03-26T21:00:34.476 INFO:teuthology.orchestra.run.smithi063.stdout: 5 134859 11.74 MB/sec execute 148 sec latency 58.196 ms 2024-03-26T21:00:35.476 INFO:teuthology.orchestra.run.smithi063.stdout: 5 135311 11.75 MB/sec execute 149 sec latency 50.613 ms 2024-03-26T21:00:35.963 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:35 smithi063 ceph-mon[24818]: pgmap v15: 97 pgs: 97 active+clean; 336 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.9 MiB/s wr, 478 op/s 2024-03-26T21:00:36.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:35 smithi174 ceph-mon[26934]: pgmap v15: 97 pgs: 97 active+clean; 336 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.9 MiB/s wr, 478 op/s 2024-03-26T21:00:36.476 INFO:teuthology.orchestra.run.smithi063.stdout: 5 135689 11.75 MB/sec execute 150 sec latency 36.971 ms 2024-03-26T21:00:37.476 INFO:teuthology.orchestra.run.smithi063.stdout: 5 136132 11.73 MB/sec execute 151 sec latency 35.575 ms 2024-03-26T21:00:37.983 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:37 smithi063 ceph-mon[24818]: pgmap v16: 97 pgs: 97 active+clean; 336 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.9 MiB/s wr, 478 op/s 2024-03-26T21:00:38.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:37 smithi174 ceph-mon[26934]: pgmap v16: 97 pgs: 97 active+clean; 336 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.9 MiB/s wr, 478 op/s 2024-03-26T21:00:38.476 INFO:teuthology.orchestra.run.smithi063.stdout: 5 136519 11.68 MB/sec execute 152 sec latency 38.545 ms 2024-03-26T21:00:39.476 INFO:teuthology.orchestra.run.smithi063.stdout: 5 136811 11.66 MB/sec execute 153 sec latency 58.025 ms 2024-03-26T21:00:39.635 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:39 smithi174 ceph-mon[26934]: pgmap v17: 97 pgs: 97 active+clean; 333 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 6.4 MiB/s wr, 597 op/s 2024-03-26T21:00:39.635 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:39 smithi174 ceph-mon[26934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' 2024-03-26T21:00:39.635 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:39 smithi174 ceph-mon[26934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' 2024-03-26T21:00:39.635 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:39 smithi174 ceph-mon[26934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-26T21:00:39.635 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:39 smithi174 ceph-mon[26934]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-26T21:00:39.635 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:39 smithi174 ceph-mon[26934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:00:39.782 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:39 smithi063 ceph-mon[24818]: pgmap v17: 97 pgs: 97 active+clean; 333 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 6.4 MiB/s wr, 597 op/s 2024-03-26T21:00:39.783 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:39 smithi063 ceph-mon[24818]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' 2024-03-26T21:00:39.783 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:39 smithi063 ceph-mon[24818]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' 2024-03-26T21:00:39.783 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:39 smithi063 ceph-mon[24818]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-26T21:00:39.783 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:39 smithi063 ceph-mon[24818]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-26T21:00:39.783 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:39 smithi063 ceph-mon[24818]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:00:40.476 INFO:teuthology.orchestra.run.smithi063.stdout: 5 137181 11.68 MB/sec execute 154 sec latency 55.752 ms 2024-03-26T21:00:40.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:40 smithi174 ceph-mon[26934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi174.sexoin", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-26T21:00:40.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:40 smithi174 ceph-mon[26934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-26T21:00:40.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:40 smithi174 ceph-mon[26934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:00:40.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:40 smithi063 ceph-mon[24818]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi174.sexoin", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-26T21:00:40.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:40 smithi063 ceph-mon[24818]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-26T21:00:40.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:40 smithi063 ceph-mon[24818]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:00:41.476 INFO:teuthology.orchestra.run.smithi063.stdout: 5 137602 11.68 MB/sec execute 155 sec latency 72.273 ms 2024-03-26T21:00:41.578 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:41 smithi063 ceph-mon[24818]: pgmap v18: 97 pgs: 97 active+clean; 333 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.9 MiB/s wr, 491 op/s 2024-03-26T21:00:41.579 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:41 smithi063 ceph-mon[24818]: Upgrade: Updating mgr.smithi174.sexoin 2024-03-26T21:00:41.579 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:41 smithi063 ceph-mon[24818]: Deploying daemon mgr.smithi174.sexoin on smithi174 2024-03-26T21:00:41.579 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:41 smithi063 ceph-mon[24818]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' 2024-03-26T21:00:41.579 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:41 smithi063 ceph-mon[24818]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:00:41.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:41 smithi174 ceph-mon[26934]: pgmap v18: 97 pgs: 97 active+clean; 333 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.9 MiB/s wr, 491 op/s 2024-03-26T21:00:41.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:41 smithi174 ceph-mon[26934]: Upgrade: Updating mgr.smithi174.sexoin 2024-03-26T21:00:41.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:41 smithi174 ceph-mon[26934]: Deploying daemon mgr.smithi174.sexoin on smithi174 2024-03-26T21:00:41.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:41 smithi174 ceph-mon[26934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' 2024-03-26T21:00:41.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:41 smithi174 ceph-mon[26934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:00:42.476 INFO:teuthology.orchestra.run.smithi063.stdout: 5 138000 11.68 MB/sec execute 156 sec latency 41.512 ms 2024-03-26T21:00:43.476 INFO:teuthology.orchestra.run.smithi063.stdout: 5 138296 11.66 MB/sec execute 157 sec latency 40.299 ms 2024-03-26T21:00:43.606 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:43 smithi063 ceph-mon[24818]: pgmap v19: 97 pgs: 97 active+clean; 333 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.5 MiB/s wr, 446 op/s 2024-03-26T21:00:43.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:43 smithi174 ceph-mon[26934]: pgmap v19: 97 pgs: 97 active+clean; 333 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.5 MiB/s wr, 446 op/s 2024-03-26T21:00:44.476 INFO:teuthology.orchestra.run.smithi063.stdout: 5 138935 11.69 MB/sec execute 158 sec latency 28.727 ms 2024-03-26T21:00:45.477 INFO:teuthology.orchestra.run.smithi063.stdout: 5 139437 11.71 MB/sec execute 159 sec latency 69.347 ms 2024-03-26T21:00:45.620 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:45 smithi063 ceph-mon[24818]: pgmap v20: 97 pgs: 97 active+clean; 337 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.7 MiB/s wr, 668 op/s 2024-03-26T21:00:45.621 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:45 smithi063 ceph-mon[24818]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' 2024-03-26T21:00:45.621 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:45 smithi063 ceph-mon[24818]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' 2024-03-26T21:00:45.621 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:45 smithi063 ceph-mon[24818]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:00:46.023 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:45 smithi174 ceph-mon[26934]: pgmap v20: 97 pgs: 97 active+clean; 337 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.7 MiB/s wr, 668 op/s 2024-03-26T21:00:46.023 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:45 smithi174 ceph-mon[26934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' 2024-03-26T21:00:46.023 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:45 smithi174 ceph-mon[26934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' 2024-03-26T21:00:46.023 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:45 smithi174 ceph-mon[26934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:00:46.476 INFO:teuthology.orchestra.run.smithi063.stdout: 5 140080 11.69 MB/sec execute 160 sec latency 58.769 ms 2024-03-26T21:00:47.476 INFO:teuthology.orchestra.run.smithi063.stdout: 5 140561 11.71 MB/sec execute 161 sec latency 36.934 ms 2024-03-26T21:00:47.639 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:47 smithi063 ceph-mon[24818]: pgmap v21: 97 pgs: 97 active+clean; 337 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.4 MiB/s wr, 434 op/s 2024-03-26T21:00:48.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:47 smithi174 ceph-mon[26934]: pgmap v21: 97 pgs: 97 active+clean; 337 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.4 MiB/s wr, 434 op/s 2024-03-26T21:00:48.476 INFO:teuthology.orchestra.run.smithi063.stdout: 5 141206 11.78 MB/sec execute 162 sec latency 42.105 ms 2024-03-26T21:00:49.477 INFO:teuthology.orchestra.run.smithi063.stdout: 5 141659 11.79 MB/sec execute 163 sec latency 64.388 ms 2024-03-26T21:00:49.681 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:49 smithi063 ceph-mon[24818]: pgmap v22: 97 pgs: 97 active+clean; 339 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.4 MiB/s wr, 739 op/s 2024-03-26T21:00:49.682 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:49 smithi063 ceph-mon[24818]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' 2024-03-26T21:00:49.682 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:49 smithi063 ceph-mon[24818]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' 2024-03-26T21:00:49.769 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:49 smithi174 ceph-mon[26934]: pgmap v22: 97 pgs: 97 active+clean; 339 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.4 MiB/s wr, 739 op/s 2024-03-26T21:00:49.769 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:49 smithi174 ceph-mon[26934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' 2024-03-26T21:00:49.769 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:49 smithi174 ceph-mon[26934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' 2024-03-26T21:00:50.476 INFO:teuthology.orchestra.run.smithi063.stdout: 5 142127 11.79 MB/sec execute 164 sec latency 44.556 ms 2024-03-26T21:00:51.344 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:51 smithi174 ceph-mon[26934]: pgmap v23: 97 pgs: 97 active+clean; 339 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.2 MiB/s wr, 526 op/s 2024-03-26T21:00:51.344 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:51 smithi174 ceph-mon[26934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' 2024-03-26T21:00:51.344 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:51 smithi174 ceph-mon[26934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' 2024-03-26T21:00:51.451 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:51 smithi063 ceph-mon[24818]: pgmap v23: 97 pgs: 97 active+clean; 339 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.2 MiB/s wr, 526 op/s 2024-03-26T21:00:51.451 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:51 smithi063 ceph-mon[24818]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' 2024-03-26T21:00:51.451 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:51 smithi063 ceph-mon[24818]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' 2024-03-26T21:00:51.477 INFO:teuthology.orchestra.run.smithi063.stdout: 5 142587 11.80 MB/sec execute 165 sec latency 27.464 ms 2024-03-26T21:00:52.477 INFO:teuthology.orchestra.run.smithi063.stdout: 5 143048 11.81 MB/sec execute 166 sec latency 38.982 ms 2024-03-26T21:00:53.466 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:53 smithi063 ceph-mon[24818]: pgmap v24: 97 pgs: 97 active+clean; 339 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.2 MiB/s wr, 526 op/s 2024-03-26T21:00:53.466 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:53 smithi063 ceph-mon[24818]: Standby manager daemon smithi174.sexoin restarted 2024-03-26T21:00:53.466 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:53 smithi063 ceph-mon[24818]: Standby manager daemon smithi174.sexoin started 2024-03-26T21:00:53.466 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:53 smithi063 ceph-mon[24818]: from='mgr.? 172.21.15.174:0/2426655209' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi174.sexoin/crt"}]: dispatch 2024-03-26T21:00:53.466 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:53 smithi063 ceph-mon[24818]: from='mgr.? 172.21.15.174:0/2426655209' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-26T21:00:53.466 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:53 smithi063 ceph-mon[24818]: from='mgr.? 172.21.15.174:0/2426655209' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi174.sexoin/key"}]: dispatch 2024-03-26T21:00:53.466 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:53 smithi063 ceph-mon[24818]: from='mgr.? 172.21.15.174:0/2426655209' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-26T21:00:53.477 INFO:teuthology.orchestra.run.smithi063.stdout: 5 143567 11.77 MB/sec execute 167 sec latency 41.239 ms 2024-03-26T21:00:53.504 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:53 smithi174 ceph-mon[26934]: pgmap v24: 97 pgs: 97 active+clean; 339 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.2 MiB/s wr, 526 op/s 2024-03-26T21:00:53.504 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:53 smithi174 ceph-mon[26934]: Standby manager daemon smithi174.sexoin restarted 2024-03-26T21:00:53.504 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:53 smithi174 ceph-mon[26934]: Standby manager daemon smithi174.sexoin started 2024-03-26T21:00:53.504 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:53 smithi174 ceph-mon[26934]: from='mgr.? 172.21.15.174:0/2426655209' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi174.sexoin/crt"}]: dispatch 2024-03-26T21:00:53.504 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:53 smithi174 ceph-mon[26934]: from='mgr.? 172.21.15.174:0/2426655209' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-26T21:00:53.504 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:53 smithi174 ceph-mon[26934]: from='mgr.? 172.21.15.174:0/2426655209' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi174.sexoin/key"}]: dispatch 2024-03-26T21:00:53.504 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:53 smithi174 ceph-mon[26934]: from='mgr.? 172.21.15.174:0/2426655209' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-26T21:00:54.477 INFO:teuthology.orchestra.run.smithi063.stdout: 5 143966 11.77 MB/sec execute 168 sec latency 28.883 ms 2024-03-26T21:00:54.482 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:54 smithi063 ceph-mon[24818]: mgrmap e33: smithi063.lxhujt(active, since 43s), standbys: smithi174.sexoin 2024-03-26T21:00:54.482 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:54 smithi063 ceph-mon[24818]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' 2024-03-26T21:00:54.483 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:54 smithi063 ceph-mon[24818]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' 2024-03-26T21:00:54.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:54 smithi174 ceph-mon[26934]: mgrmap e33: smithi063.lxhujt(active, since 43s), standbys: smithi174.sexoin 2024-03-26T21:00:54.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:54 smithi174 ceph-mon[26934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' 2024-03-26T21:00:54.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:54 smithi174 ceph-mon[26934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' 2024-03-26T21:00:55.477 INFO:teuthology.orchestra.run.smithi063.stdout: 5 144544 11.83 MB/sec execute 169 sec latency 28.869 ms 2024-03-26T21:00:55.494 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:55 smithi063 ceph-mon[24818]: pgmap v25: 97 pgs: 97 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.6 MiB/s wr, 828 op/s 2024-03-26T21:00:55.494 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:55 smithi063 ceph-mon[24818]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' 2024-03-26T21:00:55.494 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:55 smithi063 ceph-mon[24818]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' 2024-03-26T21:00:55.494 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:55 smithi063 ceph-mon[24818]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:00:55.494 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:55 smithi063 ceph-mon[24818]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T21:00:55.494 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:55 smithi063 ceph-mon[24818]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' 2024-03-26T21:00:55.494 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:55 smithi063 ceph-mon[24818]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' 2024-03-26T21:00:55.494 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:55 smithi063 ceph-mon[24818]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:00:55.494 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:55 smithi063 ceph-mon[24818]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:00:55.494 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:55 smithi063 ceph-mon[24818]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' 2024-03-26T21:00:55.494 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:55 smithi063 ceph-mon[24818]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi063.lxhujt"}]: dispatch 2024-03-26T21:00:55.495 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:55 smithi063 ceph-mon[24818]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi063.lxhujt"}]': finished 2024-03-26T21:00:55.495 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:55 smithi063 ceph-mon[24818]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi174.sexoin"}]: dispatch 2024-03-26T21:00:55.495 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:55 smithi063 ceph-mon[24818]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi174.sexoin"}]': finished 2024-03-26T21:00:55.495 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:55 smithi063 ceph-mon[24818]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-03-26T21:00:55.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:55 smithi174 ceph-mon[26934]: pgmap v25: 97 pgs: 97 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.6 MiB/s wr, 828 op/s 2024-03-26T21:00:55.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:55 smithi174 ceph-mon[26934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' 2024-03-26T21:00:55.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:55 smithi174 ceph-mon[26934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' 2024-03-26T21:00:55.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:55 smithi174 ceph-mon[26934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:00:55.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:55 smithi174 ceph-mon[26934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T21:00:55.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:55 smithi174 ceph-mon[26934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' 2024-03-26T21:00:55.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:55 smithi174 ceph-mon[26934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' 2024-03-26T21:00:55.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:55 smithi174 ceph-mon[26934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:00:55.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:55 smithi174 ceph-mon[26934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:00:55.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:55 smithi174 ceph-mon[26934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' 2024-03-26T21:00:55.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:55 smithi174 ceph-mon[26934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi063.lxhujt"}]: dispatch 2024-03-26T21:00:55.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:55 smithi174 ceph-mon[26934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi063.lxhujt"}]': finished 2024-03-26T21:00:55.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:55 smithi174 ceph-mon[26934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi174.sexoin"}]: dispatch 2024-03-26T21:00:55.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:55 smithi174 ceph-mon[26934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi174.sexoin"}]': finished 2024-03-26T21:00:55.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:55 smithi174 ceph-mon[26934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-03-26T21:00:56.477 INFO:teuthology.orchestra.run.smithi063.stdout: 5 145025 11.84 MB/sec execute 170 sec latency 45.609 ms 2024-03-26T21:00:56.497 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:56 smithi063 ceph-mon[24818]: pgmap v26: 97 pgs: 97 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 6.9 MiB/s wr, 658 op/s 2024-03-26T21:00:56.498 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:56 smithi063 ceph-mon[24818]: pgmap v27: 97 pgs: 97 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.4 MiB/s wr, 804 op/s 2024-03-26T21:00:56.498 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:56 smithi063 ceph-mon[24818]: Upgrade: Setting container_image for all mgr 2024-03-26T21:00:56.498 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:56 smithi063 ceph-mon[24818]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:00:56.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:56 smithi174 ceph-mon[26934]: pgmap v26: 97 pgs: 97 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 6.9 MiB/s wr, 658 op/s 2024-03-26T21:00:56.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:56 smithi174 ceph-mon[26934]: pgmap v27: 97 pgs: 97 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.4 MiB/s wr, 804 op/s 2024-03-26T21:00:56.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:56 smithi174 ceph-mon[26934]: Upgrade: Setting container_image for all mgr 2024-03-26T21:00:56.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:56 smithi174 ceph-mon[26934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:00:57.477 INFO:teuthology.orchestra.run.smithi063.stdout: 5 145374 11.82 MB/sec execute 171 sec latency 32.313 ms 2024-03-26T21:00:58.477 INFO:teuthology.orchestra.run.smithi063.stdout: 5 145909 11.84 MB/sec execute 172 sec latency 29.388 ms 2024-03-26T21:00:58.584 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:58 smithi063 ceph-mon[24818]: pgmap v28: 97 pgs: 97 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.4 MiB/s wr, 400 op/s 2024-03-26T21:00:58.584 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:58 smithi063 ceph-mon[24818]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' 2024-03-26T21:00:58.584 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:58 smithi063 ceph-mon[24818]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-26T21:00:58.584 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:58 smithi063 ceph-mon[24818]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-26T21:00:58.584 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:58 smithi063 ceph-mon[24818]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:00:58.710 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:58 smithi174 ceph-mon[26934]: pgmap v28: 97 pgs: 97 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.4 MiB/s wr, 400 op/s 2024-03-26T21:00:58.710 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:58 smithi174 ceph-mon[26934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' 2024-03-26T21:00:58.711 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:58 smithi174 ceph-mon[26934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-26T21:00:58.711 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:58 smithi174 ceph-mon[26934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-26T21:00:58.711 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:58 smithi174 ceph-mon[26934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:00:59.477 INFO:teuthology.orchestra.run.smithi063.stdout: 5 146298 11.84 MB/sec execute 173 sec latency 42.328 ms 2024-03-26T21:00:59.591 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:59 smithi063 ceph-mon[24818]: Upgrade: Updating mon.smithi063 2024-03-26T21:00:59.591 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:00:59 smithi063 ceph-mon[24818]: Deploying daemon mon.smithi063 on smithi063 2024-03-26T21:00:59.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:59 smithi174 ceph-mon[26934]: Upgrade: Updating mon.smithi063 2024-03-26T21:00:59.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:00:59 smithi174 ceph-mon[26934]: Deploying daemon mon.smithi063 on smithi063 2024-03-26T21:01:00.477 INFO:teuthology.orchestra.run.smithi063.stdout: 5 146620 11.82 MB/sec execute 174 sec latency 54.276 ms 2024-03-26T21:01:00.606 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:00 smithi063 ceph-mon[24818]: pgmap v29: 97 pgs: 97 active+clean; 344 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 8.2 MiB/s wr, 769 op/s 2024-03-26T21:01:00.611 INFO:teuthology.orchestra.run.smithi063.stdout:true 2024-03-26T21:01:00.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:00 smithi174 ceph-mon[26934]: pgmap v29: 97 pgs: 97 active+clean; 344 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 8.2 MiB/s wr, 769 op/s 2024-03-26T21:01:01.372 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:01 smithi063 systemd[1]: Stopping Ceph mon.smithi063 for 0ed4dd4c-ebb2-11ee-95d0-87774f69a715... 2024-03-26T21:01:01.477 INFO:teuthology.orchestra.run.smithi063.stdout: 5 146963 11.77 MB/sec execute 175 sec latency 62.826 ms 2024-03-26T21:01:01.640 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:01 smithi063 ceph-mon[24818]: from='client.14840 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:01:01.641 INFO:teuthology.orchestra.run.smithi063.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-26T21:01:01.642 INFO:teuthology.orchestra.run.smithi063.stdout:alertmanager.smithi063 smithi063 *:9093,9094 running (2m) 7s ago 11m 17.6M - 0.23.0 ba2b418f427c 6e160c94aac3 2024-03-26T21:01:01.642 INFO:teuthology.orchestra.run.smithi063.stdout:crash.smithi063 smithi063 running (11m) 7s ago 11m 6409k - 17.2.0 e1d6a67b021e 706c62f8af89 2024-03-26T21:01:01.642 INFO:teuthology.orchestra.run.smithi063.stdout:crash.smithi174 smithi174 running (10m) 12s ago 10m 6409k - 17.2.0 e1d6a67b021e 2c6bfe6d0ca6 2024-03-26T21:01:01.642 INFO:teuthology.orchestra.run.smithi063.stdout:grafana.smithi063 smithi063 *:3000 unknown 7s ago 11m - - 2024-03-26T21:01:01.642 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foofs.smithi063.tzaznz smithi063 running (5m) 7s ago 5m 36.9M - 17.2.0 e1d6a67b021e 10a989fc7954 2024-03-26T21:01:01.642 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foofs.smithi174.gitdvk smithi174 running (5m) 12s ago 5m 12.7M - 17.2.0 e1d6a67b021e 4339a5d98d1d 2024-03-26T21:01:01.642 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.smithi063.lxhujt smithi063 *:8443,9283,8765 running (73s) 7s ago 12m 495M - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 6d9d10330a90 2024-03-26T21:01:01.642 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.smithi174.sexoin smithi174 *:8443,9283,8765 running (17s) 12s ago 10m 65.9M - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 d7546233672d 2024-03-26T21:01:01.642 INFO:teuthology.orchestra.run.smithi063.stdout:mon.smithi063 smithi063 running (12m) 7s ago 12m 55.2M 2048M 17.2.0 e1d6a67b021e e6bd58b6c0f9 2024-03-26T21:01:01.642 INFO:teuthology.orchestra.run.smithi063.stdout:mon.smithi174 smithi174 running (10m) 12s ago 10m 44.8M 2048M 17.2.0 e1d6a67b021e 0f81c09f0c0a 2024-03-26T21:01:01.642 INFO:teuthology.orchestra.run.smithi063.stdout:nfs.foo.0.0.smithi063.fgmpwa smithi063 *:2049 running (5m) 7s ago 5m 267M - 4.0 e1d6a67b021e 3d6c0dd9ce51 2024-03-26T21:01:01.642 INFO:teuthology.orchestra.run.smithi063.stdout:nfs.foo.1.0.smithi174.pytptt smithi174 *:2049 running (5m) 12s ago 5m 85.8M - 4.0 e1d6a67b021e f220dbb449a3 2024-03-26T21:01:01.643 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.smithi063 smithi063 *:9100 running (109s) 7s ago 11m 9.89M - 1.3.1 1dbe0e931976 f52f41fd14c0 2024-03-26T21:01:01.643 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.smithi174 smithi174 *:9100 running (82s) 12s ago 10m 10.4M - 1.3.1 1dbe0e931976 0c027f39a4c8 2024-03-26T21:01:01.643 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0 smithi063 running (9m) 7s ago 9m 495M 1936M 17.2.0 e1d6a67b021e 761b1388d8fa 2024-03-26T21:01:01.643 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1 smithi063 running (9m) 7s ago 9m 427M 1936M 17.2.0 e1d6a67b021e 108275c17cad 2024-03-26T21:01:01.643 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2 smithi063 running (8m) 7s ago 8m 281M 1936M 17.2.0 e1d6a67b021e 107f685ad08d 2024-03-26T21:01:01.643 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3 smithi063 running (8m) 7s ago 8m 469M 1936M 17.2.0 e1d6a67b021e ab0f1a855957 2024-03-26T21:01:01.643 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4 smithi174 running (8m) 12s ago 8m 483M 2704M 17.2.0 e1d6a67b021e a0e877ecc06d 2024-03-26T21:01:01.643 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5 smithi174 running (7m) 12s ago 7m 535M 2704M 17.2.0 e1d6a67b021e 378c2aca7e71 2024-03-26T21:01:01.643 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6 smithi174 running (7m) 12s ago 7m 449M 2704M 17.2.0 e1d6a67b021e ffbb5b7ad3b6 2024-03-26T21:01:01.643 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7 smithi174 running (6m) 12s ago 6m 345M 2704M 17.2.0 e1d6a67b021e c1d57fcabb3f 2024-03-26T21:01:01.643 INFO:teuthology.orchestra.run.smithi063.stdout:prometheus.smithi063 smithi063 *:9095 running (23s) 7s ago 10m 50.6M - 2.33.4 514e6a882f6e 567eef202627 2024-03-26T21:01:01.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:01 smithi174 ceph-mon[26934]: from='client.14840 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:01:01.891 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:01 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-mon-smithi063[24814]: 2024-03-26T21:01:01.744+0000 7fe37adee700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi063 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-03-26T21:01:01.891 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:01 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-mon-smithi063[24814]: 2024-03-26T21:01:01.744+0000 7fe37adee700 -1 mon.smithi063@0(leader) e2 *** Got Signal Terminated *** 2024-03-26T21:01:01.891 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:01 smithi063 podman[66699]: 2024-03-26 21:01:01.865623562 +0000 UTC m=+0.302322772 container died e6bd58b6c0f97d6daffc88b86d5ed77e11a6cb4837d212e60cc534dd0b02c9a1 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-mon-smithi063, com.redhat.component=centos-stream-container, io.openshift.expose-services=, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, distribution-scope=public, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, build-date=2022-05-03T08:36:31.336870, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, vcs-type=git, GIT_CLEAN=True, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-03-26T21:01:02.222 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-03-26T21:01:02.222 INFO:teuthology.orchestra.run.smithi063.stdout: "mon": { 2024-03-26T21:01:02.222 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-26T21:01:02.222 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:01:02.222 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr": { 2024-03-26T21:01:02.223 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 2 2024-03-26T21:01:02.223 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:01:02.223 INFO:teuthology.orchestra.run.smithi063.stdout: "osd": { 2024-03-26T21:01:02.223 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-03-26T21:01:02.223 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:01:02.223 INFO:teuthology.orchestra.run.smithi063.stdout: "mds": { 2024-03-26T21:01:02.223 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-26T21:01:02.223 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:01:02.223 INFO:teuthology.orchestra.run.smithi063.stdout: "overall": { 2024-03-26T21:01:02.223 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-03-26T21:01:02.223 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 2 2024-03-26T21:01:02.223 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-03-26T21:01:02.223 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-03-26T21:01:02.477 INFO:teuthology.orchestra.run.smithi063.stdout: 5 147246 11.73 MB/sec execute 176 sec latency 51.890 ms 2024-03-26T21:01:02.666 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:02 smithi063 podman[66699]: 2024-03-26 21:01:02.507694704 +0000 UTC m=+0.944393904 container cleanup e6bd58b6c0f97d6daffc88b86d5ed77e11a6cb4837d212e60cc534dd0b02c9a1 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-mon-smithi063, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, version=8, name=centos-stream, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, RELEASE=HEAD, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-03-26T21:01:02.666 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:02 smithi063 bash[66699]: ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-mon-smithi063 2024-03-26T21:01:02.811 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-03-26T21:01:02.811 INFO:teuthology.orchestra.run.smithi063.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d", 2024-03-26T21:01:02.811 INFO:teuthology.orchestra.run.smithi063.stdout: "in_progress": true, 2024-03-26T21:01:02.812 INFO:teuthology.orchestra.run.smithi063.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-26T21:01:02.812 INFO:teuthology.orchestra.run.smithi063.stdout: "services_complete": [ 2024-03-26T21:01:02.812 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr" 2024-03-26T21:01:02.812 INFO:teuthology.orchestra.run.smithi063.stdout: ], 2024-03-26T21:01:02.812 INFO:teuthology.orchestra.run.smithi063.stdout: "progress": "2/23 daemons upgraded", 2024-03-26T21:01:02.812 INFO:teuthology.orchestra.run.smithi063.stdout: "message": "Currently upgrading mon daemons", 2024-03-26T21:01:02.812 INFO:teuthology.orchestra.run.smithi063.stdout: "is_paused": false 2024-03-26T21:01:02.812 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-03-26T21:01:02.922 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:02 smithi063 podman[66744]: 2024-03-26 21:01:02.794566016 +0000 UTC m=+0.927955202 container remove e6bd58b6c0f97d6daffc88b86d5ed77e11a6cb4837d212e60cc534dd0b02c9a1 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-mon-smithi063, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, vendor=Red Hat, Inc., architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, version=8, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, vcs-type=git, distribution-scope=public, release=754, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream) 2024-03-26T21:01:03.426 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:03 smithi063 systemd[1]: ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715@mon.smithi063.service: Deactivated successfully. 2024-03-26T21:01:03.426 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:03 smithi063 systemd[1]: Stopped Ceph mon.smithi063 for 0ed4dd4c-ebb2-11ee-95d0-87774f69a715. 2024-03-26T21:01:03.426 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:03 smithi063 systemd[1]: ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715@mon.smithi063.service: Consumed 16.831s CPU time. 2024-03-26T21:01:03.477 INFO:teuthology.orchestra.run.smithi063.stdout: 5 147697 11.77 MB/sec execute 177 sec latency 31.040 ms 2024-03-26T21:01:03.552 INFO:teuthology.orchestra.run.smithi063.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-26T21:01:03.552 INFO:teuthology.orchestra.run.smithi063.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-26T21:01:03.552 INFO:teuthology.orchestra.run.smithi063.stdout: daemon grafana.smithi063 on smithi063 is in unknown state 2024-03-26T21:01:03.645 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:03 smithi063 systemd[1]: Starting Ceph mon.smithi063 for 0ed4dd4c-ebb2-11ee-95d0-87774f69a715... 2024-03-26T21:01:03.932 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:03 smithi063 podman[66919]: 2024-03-26 21:01:03.780302602 +0000 UTC m=+0.031533322 image pull a9bdc3eaeb106993bcbbac35d7fe5c7b76c052021676e129f5984aed30aee063 quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d 2024-03-26T21:01:03.933 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:03 smithi063 podman[66919]: 2024-03-26 21:01:03.900050584 +0000 UTC m=+0.151281286 container create c644200344f836ef678909b7171f8409488ec090885f3b7c92402d6254342e25 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-mon-smithi063, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240326, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, ceph=True) 2024-03-26T21:01:04.478 INFO:teuthology.orchestra.run.smithi063.stdout: 5 148287 11.79 MB/sec execute 178 sec latency 30.413 ms 2024-03-26T21:01:04.692 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 podman[66919]: 2024-03-26 21:01:04.480667884 +0000 UTC m=+0.731898590 container init c644200344f836ef678909b7171f8409488ec090885f3b7c92402d6254342e25 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-mon-smithi063, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-26T21:01:04.692 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 podman[66919]: 2024-03-26 21:01:04.488303625 +0000 UTC m=+0.739534325 container start c644200344f836ef678909b7171f8409488ec090885f3b7c92402d6254342e25 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-mon-smithi063, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, org.label-schema.build-date=20240326, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-03-26T21:01:04.692 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: set uid:gid to 167:167 (ceph:ceph) 2024-03-26T21:01:04.692 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev), process ceph-mon, pid 2 2024-03-26T21:01:04.692 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: pidfile_write: ignore empty --pid-file 2024-03-26T21:01:04.692 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: load: jerasure load: lrc 2024-03-26T21:01:04.693 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: RocksDB version: 7.9.2 2024-03-26T21:01:04.693 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Git sha 0 2024-03-26T21:01:04.693 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Compile date 2024-03-26 11:42:14 2024-03-26T21:01:04.693 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: DB SUMMARY 2024-03-26T21:01:04.693 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: DB Session ID: 0DMJC9HWN2J5Y2Y30H6D 2024-03-26T21:01:04.693 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: CURRENT file: CURRENT 2024-03-26T21:01:04.693 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: IDENTITY file: IDENTITY 2024-03-26T21:01:04.693 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: MANIFEST file: MANIFEST-000015 size: 978 Bytes 2024-03-26T21:01:04.693 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi063/store.db dir, Total Num: 1, files: 000030.sst 2024-03-26T21:01:04.693 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi063/store.db: 000028.log size: 2933882 ; 2024-03-26T21:01:04.693 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.error_if_exists: 0 2024-03-26T21:01:04.693 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.create_if_missing: 0 2024-03-26T21:01:04.693 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.paranoid_checks: 1 2024-03-26T21:01:04.694 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.flush_verify_memtable_count: 1 2024-03-26T21:01:04.694 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-03-26T21:01:04.694 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-03-26T21:01:04.694 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.env: 0x55e097013c60 2024-03-26T21:01:04.694 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.fs: PosixFileSystem 2024-03-26T21:01:04.694 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.info_log: 0x55e0990699a0 2024-03-26T21:01:04.694 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.max_file_opening_threads: 16 2024-03-26T21:01:04.694 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.statistics: (nil) 2024-03-26T21:01:04.694 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.use_fsync: 0 2024-03-26T21:01:04.694 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.max_log_file_size: 0 2024-03-26T21:01:04.694 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-03-26T21:01:04.694 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.log_file_time_to_roll: 0 2024-03-26T21:01:04.694 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.keep_log_file_num: 1000 2024-03-26T21:01:04.695 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.recycle_log_file_num: 0 2024-03-26T21:01:04.695 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.allow_fallocate: 1 2024-03-26T21:01:04.695 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.allow_mmap_reads: 0 2024-03-26T21:01:04.695 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.allow_mmap_writes: 0 2024-03-26T21:01:04.695 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.use_direct_reads: 0 2024-03-26T21:01:04.695 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-03-26T21:01:04.695 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.create_missing_column_families: 0 2024-03-26T21:01:04.695 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.db_log_dir: 2024-03-26T21:01:04.695 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.wal_dir: 2024-03-26T21:01:04.695 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.table_cache_numshardbits: 6 2024-03-26T21:01:04.695 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.WAL_ttl_seconds: 0 2024-03-26T21:01:04.695 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.WAL_size_limit_MB: 0 2024-03-26T21:01:04.695 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-03-26T21:01:04.696 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-03-26T21:01:04.696 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.is_fd_close_on_exec: 1 2024-03-26T21:01:04.696 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.advise_random_on_open: 1 2024-03-26T21:01:04.696 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.db_write_buffer_size: 0 2024-03-26T21:01:04.696 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.write_buffer_manager: 0x55e09903a1e0 2024-03-26T21:01:04.696 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-03-26T21:01:04.696 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-03-26T21:01:04.696 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.use_adaptive_mutex: 0 2024-03-26T21:01:04.696 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.rate_limiter: (nil) 2024-03-26T21:01:04.696 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-03-26T21:01:04.696 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.wal_recovery_mode: 2 2024-03-26T21:01:04.696 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.enable_thread_tracking: 0 2024-03-26T21:01:04.696 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.enable_pipelined_write: 0 2024-03-26T21:01:04.697 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.unordered_write: 0 2024-03-26T21:01:04.697 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-03-26T21:01:04.697 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-03-26T21:01:04.697 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-03-26T21:01:04.697 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-03-26T21:01:04.697 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.row_cache: None 2024-03-26T21:01:04.697 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.wal_filter: None 2024-03-26T21:01:04.697 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-03-26T21:01:04.697 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.allow_ingest_behind: 0 2024-03-26T21:01:04.697 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.two_write_queues: 0 2024-03-26T21:01:04.697 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.manual_wal_flush: 0 2024-03-26T21:01:04.697 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.wal_compression: 0 2024-03-26T21:01:04.697 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.atomic_flush: 0 2024-03-26T21:01:04.698 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-03-26T21:01:04.698 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.persist_stats_to_disk: 0 2024-03-26T21:01:04.698 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.write_dbid_to_manifest: 0 2024-03-26T21:01:04.698 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.log_readahead_size: 0 2024-03-26T21:01:04.698 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-03-26T21:01:04.698 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.best_efforts_recovery: 0 2024-03-26T21:01:04.698 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-03-26T21:01:04.698 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-03-26T21:01:04.699 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.allow_data_in_errors: 0 2024-03-26T21:01:04.699 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.db_host_id: __hostname__ 2024-03-26T21:01:04.699 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.enforce_single_del_contracts: true 2024-03-26T21:01:04.699 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.max_background_jobs: 2 2024-03-26T21:01:04.699 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.max_background_compactions: -1 2024-03-26T21:01:04.699 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.max_subcompactions: 1 2024-03-26T21:01:04.699 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-03-26T21:01:04.699 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-03-26T21:01:04.699 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.delayed_write_rate : 16777216 2024-03-26T21:01:04.699 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.max_total_wal_size: 0 2024-03-26T21:01:04.699 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-03-26T21:01:04.699 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.stats_dump_period_sec: 600 2024-03-26T21:01:04.700 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.stats_persist_period_sec: 600 2024-03-26T21:01:04.700 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-03-26T21:01:04.700 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.max_open_files: -1 2024-03-26T21:01:04.700 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.bytes_per_sync: 0 2024-03-26T21:01:04.700 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.wal_bytes_per_sync: 0 2024-03-26T21:01:04.700 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.strict_bytes_per_sync: 0 2024-03-26T21:01:04.700 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.compaction_readahead_size: 0 2024-03-26T21:01:04.700 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.max_background_flushes: -1 2024-03-26T21:01:04.700 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Compression algorithms supported: 2024-03-26T21:01:04.700 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-03-26T21:01:04.700 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: kZSTD supported: 0 2024-03-26T21:01:04.700 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: kXpressCompression supported: 0 2024-03-26T21:01:04.700 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: kLZ4HCCompression supported: 1 2024-03-26T21:01:04.701 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: kZlibCompression supported: 1 2024-03-26T21:01:04.701 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: kSnappyCompression supported: 1 2024-03-26T21:01:04.701 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: kLZ4Compression supported: 1 2024-03-26T21:01:04.701 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: kBZip2Compression supported: 0 2024-03-26T21:01:04.701 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Fast CRC32 supported: Supported on x86 2024-03-26T21:01:04.701 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: DMutex implementation: pthread_mutex_t 2024-03-26T21:01:04.701 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi063/store.db/MANIFEST-000015 2024-03-26T21:01:04.701 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-03-26T21:01:04.701 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-03-26T21:01:04.701 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.merge_operator: 2024-03-26T21:01:04.701 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.compaction_filter: None 2024-03-26T21:01:04.701 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.compaction_filter_factory: None 2024-03-26T21:01:04.701 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.sst_partitioner_factory: None 2024-03-26T21:01:04.702 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.memtable_factory: SkipListFactory 2024-03-26T21:01:04.702 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.table_factory: BlockBasedTable 2024-03-26T21:01:04.702 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55e099069b60) 2024-03-26T21:01:04.702 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: cache_index_and_filter_blocks: 1 2024-03-26T21:01:04.702 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-03-26T21:01:04.702 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-03-26T21:01:04.702 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: pin_top_level_index_and_filter: 1 2024-03-26T21:01:04.702 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: index_type: 0 2024-03-26T21:01:04.702 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: data_block_index_type: 0 2024-03-26T21:01:04.702 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: index_shortening: 1 2024-03-26T21:01:04.702 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: data_block_hash_table_util_ratio: 0.750000 2024-03-26T21:01:04.702 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: checksum: 4 2024-03-26T21:01:04.702 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: no_block_cache: 0 2024-03-26T21:01:04.703 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: block_cache: 0x55e09906d090 2024-03-26T21:01:04.703 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: block_cache_name: BinnedLRUCache 2024-03-26T21:01:04.703 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: block_cache_options: 2024-03-26T21:01:04.703 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: capacity : 536870912 2024-03-26T21:01:04.703 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: num_shard_bits : 4 2024-03-26T21:01:04.703 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: strict_capacity_limit : 0 2024-03-26T21:01:04.703 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: high_pri_pool_ratio: 0.000 2024-03-26T21:01:04.703 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: block_cache_compressed: (nil) 2024-03-26T21:01:04.703 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: persistent_cache: (nil) 2024-03-26T21:01:04.703 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: block_size: 4096 2024-03-26T21:01:04.703 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: block_size_deviation: 10 2024-03-26T21:01:04.703 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: block_restart_interval: 16 2024-03-26T21:01:04.703 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: index_block_restart_interval: 1 2024-03-26T21:01:04.704 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: metadata_block_size: 4096 2024-03-26T21:01:04.704 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: partition_filters: 0 2024-03-26T21:01:04.704 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: use_delta_encoding: 1 2024-03-26T21:01:04.704 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: filter_policy: bloomfilter 2024-03-26T21:01:04.704 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: whole_key_filtering: 1 2024-03-26T21:01:04.704 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: verify_compression: 0 2024-03-26T21:01:04.704 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: read_amp_bytes_per_bit: 0 2024-03-26T21:01:04.704 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: format_version: 5 2024-03-26T21:01:04.704 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: enable_index_compression: 1 2024-03-26T21:01:04.704 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: block_align: 0 2024-03-26T21:01:04.704 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: max_auto_readahead_size: 262144 2024-03-26T21:01:04.704 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: prepopulate_block_cache: 0 2024-03-26T21:01:04.705 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: initial_auto_readahead_size: 8192 2024-03-26T21:01:04.706 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: num_file_reads_for_auto_readahead: 2 2024-03-26T21:01:04.706 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.write_buffer_size: 33554432 2024-03-26T21:01:04.706 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.max_write_buffer_number: 2 2024-03-26T21:01:04.706 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.compression: NoCompression 2024-03-26T21:01:04.706 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.bottommost_compression: Disabled 2024-03-26T21:01:04.706 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.prefix_extractor: nullptr 2024-03-26T21:01:04.706 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-03-26T21:01:04.706 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.num_levels: 7 2024-03-26T21:01:04.706 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-03-26T21:01:04.707 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-03-26T21:01:04.707 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-03-26T21:01:04.707 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-03-26T21:01:04.707 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-03-26T21:01:04.707 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-03-26T21:01:04.707 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-03-26T21:01:04.707 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-03-26T21:01:04.707 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-03-26T21:01:04.707 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-03-26T21:01:04.707 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-03-26T21:01:04.707 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-03-26T21:01:04.707 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.compression_opts.window_bits: -14 2024-03-26T21:01:04.707 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.compression_opts.level: 32767 2024-03-26T21:01:04.708 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.compression_opts.strategy: 0 2024-03-26T21:01:04.708 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-03-26T21:01:04.708 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-03-26T21:01:04.708 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-03-26T21:01:04.708 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-03-26T21:01:04.708 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.compression_opts.enabled: false 2024-03-26T21:01:04.708 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-03-26T21:01:04.708 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-03-26T21:01:04.708 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-03-26T21:01:04.708 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-03-26T21:01:04.708 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.target_file_size_base: 67108864 2024-03-26T21:01:04.708 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.target_file_size_multiplier: 1 2024-03-26T21:01:04.708 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-03-26T21:01:04.708 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-03-26T21:01:04.709 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-03-26T21:01:04.709 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-03-26T21:01:04.709 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-03-26T21:01:04.709 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-03-26T21:01:04.709 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-03-26T21:01:04.709 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-03-26T21:01:04.709 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-03-26T21:01:04.709 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-03-26T21:01:04.709 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-03-26T21:01:04.709 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-03-26T21:01:04.709 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-03-26T21:01:04.709 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.arena_block_size: 1048576 2024-03-26T21:01:04.709 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-03-26T21:01:04.710 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-03-26T21:01:04.710 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.disable_auto_compactions: 0 2024-03-26T21:01:04.710 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-03-26T21:01:04.710 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-03-26T21:01:04.710 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-03-26T21:01:04.710 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-03-26T21:01:04.710 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-03-26T21:01:04.710 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-03-26T21:01:04.710 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-03-26T21:01:04.710 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-03-26T21:01:04.710 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-03-26T21:01:04.710 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-03-26T21:01:04.710 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-03-26T21:01:04.711 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.inplace_update_support: 0 2024-03-26T21:01:04.711 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.inplace_update_num_locks: 10000 2024-03-26T21:01:04.711 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-03-26T21:01:04.711 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-03-26T21:01:04.711 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.memtable_huge_page_size: 0 2024-03-26T21:01:04.711 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.bloom_locality: 0 2024-03-26T21:01:04.711 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.max_successive_merges: 0 2024-03-26T21:01:04.711 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.optimize_filters_for_hits: 0 2024-03-26T21:01:04.711 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.paranoid_file_checks: 0 2024-03-26T21:01:04.711 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.force_consistency_checks: 1 2024-03-26T21:01:04.711 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.report_bg_io_stats: 0 2024-03-26T21:01:04.711 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.ttl: 2592000 2024-03-26T21:01:04.711 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.periodic_compaction_seconds: 0 2024-03-26T21:01:04.712 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-03-26T21:01:04.712 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-03-26T21:01:04.712 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.enable_blob_files: false 2024-03-26T21:01:04.712 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.min_blob_size: 0 2024-03-26T21:01:04.712 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.blob_file_size: 268435456 2024-03-26T21:01:04.712 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.blob_compression_type: NoCompression 2024-03-26T21:01:04.712 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.enable_blob_garbage_collection: false 2024-03-26T21:01:04.712 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-03-26T21:01:04.712 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-03-26T21:01:04.712 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-03-26T21:01:04.712 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.blob_file_starting_level: 0 2024-03-26T21:01:04.712 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-03-26T21:01:04.713 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 30.sst 2024-03-26T21:01:04.713 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-03-26T21:01:04.713 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi063/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 32, last_sequence is 11796, log_number is 28,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-03-26T21:01:04.713 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 28 2024-03-26T21:01:04.713 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 55255b81-e890-4c25-9d1d-44215cfc3b71 2024-03-26T21:01:04.713 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: EVENT_LOG_v1 {"time_micros": 1711486864530666, "job": 1, "event": "recovery_started", "wal_files": [28]} 2024-03-26T21:01:04.713 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #28 mode 2 2024-03-26T21:01:04.713 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: EVENT_LOG_v1 {"time_micros": 1711486864541697, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 33, "file_size": 2699504, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 11797, "largest_seqno": 13009, "table_properties": {"data_size": 2693401, "index_size": 3714, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1349, "raw_key_size": 11854, "raw_average_key_size": 22, "raw_value_size": 2682051, "raw_average_value_size": 5197, "num_data_blocks": 169, "num_entries": 516, "num_filter_entries": 516, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1711486864, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "55255b81-e890-4c25-9d1d-44215cfc3b71", "db_session_id": "0DMJC9HWN2J5Y2Y30H6D", "orig_file_number": 33, "seqno_to_time_mapping": "N/A"}} 2024-03-26T21:01:04.713 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: EVENT_LOG_v1 {"time_micros": 1711486864541788, "job": 1, "event": "recovery_finished"} 2024-03-26T21:01:04.713 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: [db/version_set.cc:5047] Creating manifest 35 2024-03-26T21:01:04.713 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-03-26T21:01:04.713 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-smithi063/store.db/000028.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-03-26T21:01:04.713 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55e09915c000 2024-03-26T21:01:04.714 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: DB pointer 0x55e099146000 2024-03-26T21:01:04.714 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-03-26T21:01:04.714 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-03-26T21:01:04.714 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: ** DB Stats ** 2024-03-26T21:01:04.714 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-03-26T21:01:04.714 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-03-26T21:01:04.714 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-03-26T21:01:04.714 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-26T21:01:04.714 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-03-26T21:01:04.714 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-03-26T21:01:04.714 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-26T21:01:04.714 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: 2024-03-26T21:01:04.714 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: ** Compaction Stats [default] ** 2024-03-26T21:01:04.715 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-03-26T21:01:04.715 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-03-26T21:01:04.715 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: L0 1/0 2.57 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 532.9 0.00 0.00 1 0.005 0 0 0.0 0.0 2024-03-26T21:01:04.715 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: L6 1/0 6.20 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-03-26T21:01:04.715 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: Sum 2/0 8.78 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 532.9 0.00 0.00 1 0.005 0 0 0.0 0.0 2024-03-26T21:01:04.715 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 532.9 0.00 0.00 1 0.005 0 0 0.0 0.0 2024-03-26T21:01:04.715 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: 2024-03-26T21:01:04.715 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: ** Compaction Stats [default] ** 2024-03-26T21:01:04.715 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-03-26T21:01:04.715 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-03-26T21:01:04.715 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 532.9 0.00 0.00 1 0.005 0 0 0.0 0.0 2024-03-26T21:01:04.715 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: 2024-03-26T21:01:04.715 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-03-26T21:01:04.716 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: 2024-03-26T21:01:04.716 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-03-26T21:01:04.716 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: Flush(GB): cumulative 0.003, interval 0.003 2024-03-26T21:01:04.716 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-03-26T21:01:04.716 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-03-26T21:01:04.716 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-03-26T21:01:04.716 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: AddFile(Keys): cumulative 0, interval 0 2024-03-26T21:01:04.716 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: Cumulative compaction: 0.00 GB write, 173.10 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-26T21:01:04.716 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: Interval compaction: 0.00 GB write, 173.10 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-26T21:01:04.716 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-03-26T21:01:04.716 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: Block cache BinnedLRUCache@0x55e09906d090#2 capacity: 512.00 MB usage: 5.48 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.2e-05 secs_since: 0 2024-03-26T21:01:04.716 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.41 KB,0.000268221%) IndexBlock(1,4.08 KB,0.000777841%) Misc(1,0.00 KB,0%) 2024-03-26T21:01:04.716 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: 2024-03-26T21:01:04.717 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: ** File Read Latency Histogram By Level [default] ** 2024-03-26T21:01:04.717 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: starting mon.smithi063 rank 0 at public addrs [v2:172.21.15.63:3300/0,v1:172.21.15.63:6789/0] at bind addrs [v2:172.21.15.63:3300/0,v1:172.21.15.63:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi063 fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 2024-03-26T21:01:04.717 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: mon.smithi063@-1(???) e2 preinit fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 2024-03-26T21:01:04.717 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: mon.smithi063@-1(???).mds e7 new map 2024-03-26T21:01:04.717 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: mon.smithi063@-1(???).mds e7 print_map 2024-03-26T21:01:04.717 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: e7 2024-03-26T21:01:04.717 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-03-26T21:01:04.717 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-03-26T21:01:04.717 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: legacy client fscid: 1 2024-03-26T21:01:04.717 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: 2024-03-26T21:01:04.717 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: Filesystem 'foofs' (1) 2024-03-26T21:01:04.717 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: fs_name foofs 2024-03-26T21:01:04.718 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: epoch 5 2024-03-26T21:01:04.718 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-03-26T21:01:04.718 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: created 2024-03-26T20:55:32.397790+0000 2024-03-26T21:01:04.718 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: modified 2024-03-26T20:55:36.411042+0000 2024-03-26T21:01:04.718 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: tableserver 0 2024-03-26T21:01:04.718 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: root 0 2024-03-26T21:01:04.718 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: session_timeout 60 2024-03-26T21:01:04.718 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: session_autoclose 300 2024-03-26T21:01:04.719 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: max_file_size 1099511627776 2024-03-26T21:01:04.719 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: max_xattr_size 65536 2024-03-26T21:01:04.719 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: required_client_features {} 2024-03-26T21:01:04.719 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: last_failure 0 2024-03-26T21:01:04.719 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: last_failure_osd_epoch 0 2024-03-26T21:01:04.719 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-03-26T21:01:04.719 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: max_mds 1 2024-03-26T21:01:04.719 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: in 0 2024-03-26T21:01:04.719 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: up {0=14474} 2024-03-26T21:01:04.719 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: failed 2024-03-26T21:01:04.719 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: damaged 2024-03-26T21:01:04.719 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: stopped 2024-03-26T21:01:04.719 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: data_pools [3] 2024-03-26T21:01:04.720 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: metadata_pool 2 2024-03-26T21:01:04.720 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: inline_data disabled 2024-03-26T21:01:04.720 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: balancer 2024-03-26T21:01:04.720 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: bal_rank_mask -1 2024-03-26T21:01:04.720 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: standby_count_wanted 1 2024-03-26T21:01:04.720 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: qdb_cluster leader: 0 members: 2024-03-26T21:01:04.720 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: [mds.foofs.smithi063.tzaznz{0:14474} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.63:6834/1380786213,v1:172.21.15.63:6835/1380786213] compat {c=[1],r=[1],i=[7ff]}] 2024-03-26T21:01:04.720 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: 2024-03-26T21:01:04.720 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: 2024-03-26T21:01:04.720 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: Standby daemons: 2024-03-26T21:01:04.720 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: 2024-03-26T21:01:04.720 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout: [mds.foofs.smithi174.gitdvk{-1:24307} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.174:6832/2611248035,v1:172.21.15.174:6833/2611248035] compat {c=[1],r=[1],i=[7ff]}] 2024-03-26T21:01:04.720 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: mon.smithi063@-1(???).osd e70 crush map has features 3314933000854323200, adjusting msgr requires 2024-03-26T21:01:04.721 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: mon.smithi063@-1(???).osd e70 crush map has features 432629239337189376, adjusting msgr requires 2024-03-26T21:01:04.721 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: mon.smithi063@-1(???).osd e70 crush map has features 432629239337189376, adjusting msgr requires 2024-03-26T21:01:04.721 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: mon.smithi063@-1(???).osd e70 crush map has features 432629239337189376, adjusting msgr requires 2024-03-26T21:01:04.721 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 ceph-mon[66934]: mon.smithi063@-1(???).paxosservice(auth 1..29) refresh upgraded, format 0 -> 3 2024-03-26T21:01:04.721 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 bash[66919]: c644200344f836ef678909b7171f8409488ec090885f3b7c92402d6254342e25 2024-03-26T21:01:04.721 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:04 smithi063 systemd[1]: Started Ceph mon.smithi063 for 0ed4dd4c-ebb2-11ee-95d0-87774f69a715. 2024-03-26T21:01:05.448 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:05 smithi063 ceph-mon[66934]: from='client.14844 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:01:05.448 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:05 smithi063 ceph-mon[66934]: from='client.14848 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:01:05.448 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:05 smithi063 ceph-mon[66934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mon metadata", "id": "smithi063"}]: dispatch 2024-03-26T21:01:05.448 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:05 smithi063 ceph-mon[66934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mon metadata", "id": "smithi174"}]: dispatch 2024-03-26T21:01:05.448 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:05 smithi063 ceph-mon[66934]: from='client.? 172.21.15.63:0/1887043967' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:01:05.449 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:05 smithi063 ceph-mon[66934]: from='client.24617 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:01:05.449 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:05 smithi063 ceph-mon[66934]: pgmap v31: 97 pgs: 97 active+clean; 341 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.6 MiB/s wr, 680 op/s 2024-03-26T21:01:05.449 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:05 smithi063 ceph-mon[66934]: from='client.? 172.21.15.63:0/2921477359' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-26T21:01:05.449 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:05 smithi063 ceph-mon[66934]: mon.smithi063 calling monitor election 2024-03-26T21:01:05.449 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:05 smithi063 ceph-mon[66934]: mon.smithi063 is new leader, mons smithi063,smithi174 in quorum (ranks 0,1) 2024-03-26T21:01:05.449 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:05 smithi063 ceph-mon[66934]: monmap epoch 2 2024-03-26T21:01:05.449 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:05 smithi063 ceph-mon[66934]: fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 2024-03-26T21:01:05.449 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:05 smithi063 ceph-mon[66934]: last_changed 2024-03-26T20:50:24.841910+0000 2024-03-26T21:01:05.449 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:05 smithi063 ceph-mon[66934]: created 2024-03-26T20:48:04.639095+0000 2024-03-26T21:01:05.449 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:05 smithi063 ceph-mon[66934]: min_mon_release 17 (quincy) 2024-03-26T21:01:05.449 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:05 smithi063 ceph-mon[66934]: election_strategy: 1 2024-03-26T21:01:05.449 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:05 smithi063 ceph-mon[66934]: 0: [v2:172.21.15.63:3300/0,v1:172.21.15.63:6789/0] mon.smithi063 2024-03-26T21:01:05.449 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:05 smithi063 ceph-mon[66934]: 1: [v2:172.21.15.174:3300/0,v1:172.21.15.174:6789/0] mon.smithi174 2024-03-26T21:01:05.449 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:05 smithi063 ceph-mon[66934]: fsmap foofs:1 {0=foofs.smithi063.tzaznz=up:active} 1 up:standby 2024-03-26T21:01:05.449 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:05 smithi063 ceph-mon[66934]: osdmap e70: 8 total, 8 up, 8 in 2024-03-26T21:01:05.450 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:05 smithi063 ceph-mon[66934]: mgrmap e33: smithi063.lxhujt(active, since 55s), standbys: smithi174.sexoin 2024-03-26T21:01:05.450 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:05 smithi063 ceph-mon[66934]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-26T21:01:05.450 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:05 smithi063 ceph-mon[66934]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-26T21:01:05.450 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:05 smithi063 ceph-mon[66934]: daemon grafana.smithi063 on smithi063 is in unknown state 2024-03-26T21:01:05.450 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:05 smithi063 ceph-mon[66934]: from='mgr.24529 ' entity='' 2024-03-26T21:01:05.450 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:05 smithi063 ceph-mon[66934]: mgrmap e34: smithi063.lxhujt(active, since 55s), standbys: smithi174.sexoin 2024-03-26T21:01:05.478 INFO:teuthology.orchestra.run.smithi063.stdout: 5 148721 11.80 MB/sec execute 179 sec latency 32.228 ms 2024-03-26T21:01:05.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:05 smithi174 ceph-mon[26934]: from='client.14844 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:01:05.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:05 smithi174 ceph-mon[26934]: from='client.14848 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:01:05.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:05 smithi174 ceph-mon[26934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mon metadata", "id": "smithi063"}]: dispatch 2024-03-26T21:01:05.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:05 smithi174 ceph-mon[26934]: from='mgr.24529 172.21.15.63:0/1153377097' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mon metadata", "id": "smithi174"}]: dispatch 2024-03-26T21:01:05.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:05 smithi174 ceph-mon[26934]: from='client.? 172.21.15.63:0/1887043967' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:01:05.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:05 smithi174 ceph-mon[26934]: from='client.24617 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:01:05.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:05 smithi174 ceph-mon[26934]: pgmap v31: 97 pgs: 97 active+clean; 341 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.6 MiB/s wr, 680 op/s 2024-03-26T21:01:05.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:05 smithi174 ceph-mon[26934]: from='client.? 172.21.15.63:0/2921477359' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-26T21:01:05.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:05 smithi174 ceph-mon[26934]: mon.smithi063 calling monitor election 2024-03-26T21:01:05.550 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:05 smithi174 ceph-mon[26934]: mon.smithi063 is new leader, mons smithi063,smithi174 in quorum (ranks 0,1) 2024-03-26T21:01:05.550 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:05 smithi174 ceph-mon[26934]: monmap epoch 2 2024-03-26T21:01:05.550 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:05 smithi174 ceph-mon[26934]: fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 2024-03-26T21:01:05.550 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:05 smithi174 ceph-mon[26934]: last_changed 2024-03-26T20:50:24.841910+0000 2024-03-26T21:01:05.550 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:05 smithi174 ceph-mon[26934]: created 2024-03-26T20:48:04.639095+0000 2024-03-26T21:01:05.550 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:05 smithi174 ceph-mon[26934]: min_mon_release 17 (quincy) 2024-03-26T21:01:05.550 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:05 smithi174 ceph-mon[26934]: election_strategy: 1 2024-03-26T21:01:05.550 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:05 smithi174 ceph-mon[26934]: 0: [v2:172.21.15.63:3300/0,v1:172.21.15.63:6789/0] mon.smithi063 2024-03-26T21:01:05.550 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:05 smithi174 ceph-mon[26934]: 1: [v2:172.21.15.174:3300/0,v1:172.21.15.174:6789/0] mon.smithi174 2024-03-26T21:01:05.550 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:05 smithi174 ceph-mon[26934]: fsmap foofs:1 {0=foofs.smithi063.tzaznz=up:active} 1 up:standby 2024-03-26T21:01:05.550 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:05 smithi174 ceph-mon[26934]: osdmap e70: 8 total, 8 up, 8 in 2024-03-26T21:01:05.550 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:05 smithi174 ceph-mon[26934]: mgrmap e33: smithi063.lxhujt(active, since 55s), standbys: smithi174.sexoin 2024-03-26T21:01:05.550 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:05 smithi174 ceph-mon[26934]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-26T21:01:05.550 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:05 smithi174 ceph-mon[26934]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-26T21:01:05.550 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:05 smithi174 ceph-mon[26934]: daemon grafana.smithi063 on smithi063 is in unknown state 2024-03-26T21:01:05.551 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:05 smithi174 ceph-mon[26934]: from='mgr.24529 ' entity='' 2024-03-26T21:01:05.551 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:05 smithi174 ceph-mon[26934]: mgrmap e34: smithi063.lxhujt(active, since 55s), standbys: smithi174.sexoin 2024-03-26T21:01:06.478 INFO:teuthology.orchestra.run.smithi063.stdout: 5 149118 11.79 MB/sec execute 180 sec latency 31.611 ms 2024-03-26T21:01:07.217 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:07 smithi063 ceph-mon[66934]: mgrmap e35: smithi063.lxhujt(active, since 56s), standbys: smithi174.sexoin 2024-03-26T21:01:07.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:07 smithi174 ceph-mon[26934]: mgrmap e35: smithi063.lxhujt(active, since 56s), standbys: smithi174.sexoin 2024-03-26T21:01:07.478 INFO:teuthology.orchestra.run.smithi063.stdout: 5 149515 11.78 MB/sec execute 181 sec latency 43.579 ms 2024-03-26T21:01:08.478 INFO:teuthology.orchestra.run.smithi063.stdout: 5 149824 11.78 MB/sec execute 182 sec latency 56.514 ms 2024-03-26T21:01:09.478 INFO:teuthology.orchestra.run.smithi063.stdout: 5 150152 11.76 MB/sec execute 183 sec latency 62.647 ms 2024-03-26T21:01:10.478 INFO:teuthology.orchestra.run.smithi063.stdout: 5 150551 11.71 MB/sec execute 184 sec latency 35.716 ms 2024-03-26T21:01:11.479 INFO:teuthology.orchestra.run.smithi063.stdout: 5 150830 11.68 MB/sec execute 185 sec latency 64.383 ms 2024-03-26T21:01:12.478 INFO:teuthology.orchestra.run.smithi063.stdout: 5 151090 11.66 MB/sec execute 186 sec latency 39.179 ms 2024-03-26T21:01:13.478 INFO:teuthology.orchestra.run.smithi063.stdout: 5 151444 11.68 MB/sec execute 187 sec latency 66.712 ms 2024-03-26T21:01:13.610 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:13 smithi063 ceph-mon[66934]: Standby manager daemon smithi174.sexoin restarted 2024-03-26T21:01:13.610 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:13 smithi063 ceph-mon[66934]: Standby manager daemon smithi174.sexoin started 2024-03-26T21:01:13.610 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:13 smithi063 ceph-mon[66934]: from='mgr.? 172.21.15.174:0/3927347127' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi174.sexoin/crt"}]: dispatch 2024-03-26T21:01:13.610 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:13 smithi063 ceph-mon[66934]: from='mgr.? 172.21.15.174:0/3927347127' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-26T21:01:13.610 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:13 smithi063 ceph-mon[66934]: from='mgr.? 172.21.15.174:0/3927347127' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi174.sexoin/key"}]: dispatch 2024-03-26T21:01:13.611 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:13 smithi063 ceph-mon[66934]: from='mgr.? 172.21.15.174:0/3927347127' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-26T21:01:13.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:13 smithi174 ceph-mon[26934]: Standby manager daemon smithi174.sexoin restarted 2024-03-26T21:01:13.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:13 smithi174 ceph-mon[26934]: Standby manager daemon smithi174.sexoin started 2024-03-26T21:01:13.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:13 smithi174 ceph-mon[26934]: from='mgr.? 172.21.15.174:0/3927347127' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi174.sexoin/crt"}]: dispatch 2024-03-26T21:01:13.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:13 smithi174 ceph-mon[26934]: from='mgr.? 172.21.15.174:0/3927347127' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-26T21:01:13.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:13 smithi174 ceph-mon[26934]: from='mgr.? 172.21.15.174:0/3927347127' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi174.sexoin/key"}]: dispatch 2024-03-26T21:01:13.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:13 smithi174 ceph-mon[26934]: from='mgr.? 172.21.15.174:0/3927347127' entity='mgr.smithi174.sexoin' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-26T21:01:14.478 INFO:teuthology.orchestra.run.smithi063.stdout: 5 151849 11.68 MB/sec execute 188 sec latency 44.835 ms 2024-03-26T21:01:14.621 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:14 smithi063 ceph-mon[66934]: mgrmap e36: smithi063.lxhujt(active, since 63s), standbys: smithi174.sexoin 2024-03-26T21:01:14.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:14 smithi174 ceph-mon[26934]: mgrmap e36: smithi063.lxhujt(active, since 63s), standbys: smithi174.sexoin 2024-03-26T21:01:15.478 INFO:teuthology.orchestra.run.smithi063.stdout: 5 152344 11.68 MB/sec execute 189 sec latency 38.892 ms 2024-03-26T21:01:15.628 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:15 smithi063 ceph-mon[66934]: Active manager daemon smithi063.lxhujt restarted 2024-03-26T21:01:15.629 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:15 smithi063 ceph-mon[66934]: Activating manager daemon smithi063.lxhujt 2024-03-26T21:01:15.629 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:15 smithi063 ceph-mon[66934]: osdmap e71: 8 total, 8 up, 8 in 2024-03-26T21:01:15.629 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:15 smithi063 ceph-mon[66934]: mgrmap e37: smithi063.lxhujt(active, starting, since 0.0158768s), standbys: smithi174.sexoin 2024-03-26T21:01:15.629 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:15 smithi063 ceph-mon[66934]: from='mgr.34104 172.21.15.63:0/1722509725' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mon metadata", "id": "smithi063"}]: dispatch 2024-03-26T21:01:15.629 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:15 smithi063 ceph-mon[66934]: from='mgr.34104 172.21.15.63:0/1722509725' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mon metadata", "id": "smithi174"}]: dispatch 2024-03-26T21:01:15.629 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:15 smithi063 ceph-mon[66934]: from='mgr.34104 172.21.15.63:0/1722509725' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi063.tzaznz"}]: dispatch 2024-03-26T21:01:15.629 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:15 smithi063 ceph-mon[66934]: from='mgr.34104 172.21.15.63:0/1722509725' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi174.gitdvk"}]: dispatch 2024-03-26T21:01:15.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:15 smithi174 ceph-mon[26934]: Active manager daemon smithi063.lxhujt restarted 2024-03-26T21:01:15.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:15 smithi174 ceph-mon[26934]: Activating manager daemon smithi063.lxhujt 2024-03-26T21:01:15.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:15 smithi174 ceph-mon[26934]: osdmap e71: 8 total, 8 up, 8 in 2024-03-26T21:01:15.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:15 smithi174 ceph-mon[26934]: mgrmap e37: smithi063.lxhujt(active, starting, since 0.0158768s), standbys: smithi174.sexoin 2024-03-26T21:01:15.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:15 smithi174 ceph-mon[26934]: from='mgr.34104 172.21.15.63:0/1722509725' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mon metadata", "id": "smithi063"}]: dispatch 2024-03-26T21:01:15.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:15 smithi174 ceph-mon[26934]: from='mgr.34104 172.21.15.63:0/1722509725' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mon metadata", "id": "smithi174"}]: dispatch 2024-03-26T21:01:15.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:15 smithi174 ceph-mon[26934]: from='mgr.34104 172.21.15.63:0/1722509725' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi063.tzaznz"}]: dispatch 2024-03-26T21:01:15.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:15 smithi174 ceph-mon[26934]: from='mgr.34104 172.21.15.63:0/1722509725' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi174.gitdvk"}]: dispatch 2024-03-26T21:01:16.478 INFO:teuthology.orchestra.run.smithi063.stdout: 5 152898 11.70 MB/sec execute 190 sec latency 26.766 ms 2024-03-26T21:01:17.478 INFO:teuthology.orchestra.run.smithi063.stdout: 5 153338 11.73 MB/sec execute 191 sec latency 30.308 ms 2024-03-26T21:01:18.478 INFO:teuthology.orchestra.run.smithi063.stdout: 5 153739 11.71 MB/sec execute 192 sec latency 49.743 ms 2024-03-26T21:01:19.478 INFO:teuthology.orchestra.run.smithi063.stdout: 5 154213 11.67 MB/sec execute 193 sec latency 34.725 ms 2024-03-26T21:01:20.479 INFO:teuthology.orchestra.run.smithi063.stdout: 5 154479 11.65 MB/sec execute 194 sec latency 43.662 ms 2024-03-26T21:01:21.478 INFO:teuthology.orchestra.run.smithi063.stdout: 5 154889 11.68 MB/sec execute 195 sec latency 63.931 ms 2024-03-26T21:01:22.478 INFO:teuthology.orchestra.run.smithi063.stdout: 5 155313 11.68 MB/sec execute 196 sec latency 66.743 ms 2024-03-26T21:01:23.478 INFO:teuthology.orchestra.run.smithi063.stdout: 5 155783 11.69 MB/sec execute 197 sec latency 31.251 ms 2024-03-26T21:01:24.479 INFO:teuthology.orchestra.run.smithi063.stdout: 5 156108 11.67 MB/sec execute 198 sec latency 39.904 ms 2024-03-26T21:01:25.479 INFO:teuthology.orchestra.run.smithi063.stdout: 5 156507 11.65 MB/sec execute 199 sec latency 45.819 ms 2024-03-26T21:01:25.787 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:25 smithi063 ceph-mon[66934]: Active manager daemon smithi063.lxhujt restarted 2024-03-26T21:01:25.787 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:25 smithi063 ceph-mon[66934]: Activating manager daemon smithi063.lxhujt 2024-03-26T21:01:25.787 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:25 smithi063 ceph-mon[66934]: osdmap e72: 8 total, 8 up, 8 in 2024-03-26T21:01:25.787 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:25 smithi063 ceph-mon[66934]: mgrmap e38: smithi063.lxhujt(active, starting, since 0.00840923s), standbys: smithi174.sexoin 2024-03-26T21:01:25.788 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:25 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mon metadata", "id": "smithi063"}]: dispatch 2024-03-26T21:01:25.788 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:25 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mon metadata", "id": "smithi174"}]: dispatch 2024-03-26T21:01:25.788 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:25 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi063.tzaznz"}]: dispatch 2024-03-26T21:01:25.788 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:25 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi174.gitdvk"}]: dispatch 2024-03-26T21:01:25.788 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:25 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mgr metadata", "who": "smithi063.lxhujt", "id": "smithi063.lxhujt"}]: dispatch 2024-03-26T21:01:25.788 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:25 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mgr metadata", "who": "smithi174.sexoin", "id": "smithi174.sexoin"}]: dispatch 2024-03-26T21:01:25.788 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:25 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-26T21:01:25.788 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:25 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-26T21:01:25.788 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:25 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-26T21:01:25.788 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:25 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-26T21:01:25.788 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:25 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-26T21:01:25.788 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:25 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-26T21:01:25.788 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:25 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-26T21:01:25.788 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:25 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-26T21:01:25.789 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:25 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-26T21:01:25.789 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:25 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-26T21:01:25.789 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:25 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-26T21:01:25.883 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:25 smithi174 ceph-mon[26934]: Active manager daemon smithi063.lxhujt restarted 2024-03-26T21:01:25.883 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:25 smithi174 ceph-mon[26934]: Activating manager daemon smithi063.lxhujt 2024-03-26T21:01:25.883 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:25 smithi174 ceph-mon[26934]: osdmap e72: 8 total, 8 up, 8 in 2024-03-26T21:01:25.883 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:25 smithi174 ceph-mon[26934]: mgrmap e38: smithi063.lxhujt(active, starting, since 0.00840923s), standbys: smithi174.sexoin 2024-03-26T21:01:25.883 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:25 smithi174 ceph-mon[26934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mon metadata", "id": "smithi063"}]: dispatch 2024-03-26T21:01:25.883 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:25 smithi174 ceph-mon[26934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mon metadata", "id": "smithi174"}]: dispatch 2024-03-26T21:01:25.883 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:25 smithi174 ceph-mon[26934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi063.tzaznz"}]: dispatch 2024-03-26T21:01:25.883 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:25 smithi174 ceph-mon[26934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi174.gitdvk"}]: dispatch 2024-03-26T21:01:25.884 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:25 smithi174 ceph-mon[26934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mgr metadata", "who": "smithi063.lxhujt", "id": "smithi063.lxhujt"}]: dispatch 2024-03-26T21:01:25.884 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:25 smithi174 ceph-mon[26934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mgr metadata", "who": "smithi174.sexoin", "id": "smithi174.sexoin"}]: dispatch 2024-03-26T21:01:25.884 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:25 smithi174 ceph-mon[26934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-26T21:01:25.884 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:25 smithi174 ceph-mon[26934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-26T21:01:25.884 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:25 smithi174 ceph-mon[26934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-26T21:01:25.884 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:25 smithi174 ceph-mon[26934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-26T21:01:25.884 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:25 smithi174 ceph-mon[26934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-26T21:01:25.884 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:25 smithi174 ceph-mon[26934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-26T21:01:25.884 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:25 smithi174 ceph-mon[26934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-26T21:01:25.884 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:25 smithi174 ceph-mon[26934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-26T21:01:25.884 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:25 smithi174 ceph-mon[26934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-26T21:01:25.884 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:25 smithi174 ceph-mon[26934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-26T21:01:25.884 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:25 smithi174 ceph-mon[26934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-26T21:01:26.478 INFO:teuthology.orchestra.run.smithi063.stdout: 5 156760 11.65 MB/sec execute 200 sec latency 64.554 ms 2024-03-26T21:01:26.819 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:26 smithi063 ceph-mon[66934]: Manager daemon smithi063.lxhujt is now available 2024-03-26T21:01:26.819 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:26 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:01:26.819 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:26 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:01:26.819 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:26 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi063.lxhujt/mirror_snapshot_schedule"}]: dispatch 2024-03-26T21:01:26.819 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:26 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi063.lxhujt/trash_purge_schedule"}]: dispatch 2024-03-26T21:01:27.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:26 smithi174 ceph-mon[26934]: Manager daemon smithi063.lxhujt is now available 2024-03-26T21:01:27.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:26 smithi174 ceph-mon[26934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:01:27.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:26 smithi174 ceph-mon[26934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:01:27.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:26 smithi174 ceph-mon[26934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi063.lxhujt/mirror_snapshot_schedule"}]: dispatch 2024-03-26T21:01:27.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:26 smithi174 ceph-mon[26934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi063.lxhujt/trash_purge_schedule"}]: dispatch 2024-03-26T21:01:27.479 INFO:teuthology.orchestra.run.smithi063.stdout: 5 157012 11.63 MB/sec execute 201 sec latency 63.132 ms 2024-03-26T21:01:27.595 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:27 smithi063 ceph-mon[66934]: mgrmap e39: smithi063.lxhujt(active, since 1.09363s), standbys: smithi174.sexoin 2024-03-26T21:01:27.596 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:27 smithi063 ceph-mon[66934]: pgmap v3: 97 pgs: 97 active+clean; 345 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-03-26T21:01:27.895 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:27 smithi174 ceph-mon[26934]: mgrmap e39: smithi063.lxhujt(active, since 1.09363s), standbys: smithi174.sexoin 2024-03-26T21:01:27.895 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:27 smithi174 ceph-mon[26934]: pgmap v3: 97 pgs: 97 active+clean; 345 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-03-26T21:01:28.479 INFO:teuthology.orchestra.run.smithi063.stdout: 5 157365 11.60 MB/sec execute 202 sec latency 54.695 ms 2024-03-26T21:01:28.621 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:28 smithi063 ceph-mon[66934]: [26/Mar/2024:21:01:27] ENGINE Bus STARTING 2024-03-26T21:01:28.621 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:28 smithi063 ceph-mon[66934]: pgmap v4: 97 pgs: 97 active+clean; 345 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-03-26T21:01:28.621 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:28 smithi063 ceph-mon[66934]: [26/Mar/2024:21:01:27] ENGINE Serving on https://172.21.15.63:7150 2024-03-26T21:01:28.621 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:28 smithi063 ceph-mon[66934]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-03-26T21:01:28.621 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:28 smithi063 ceph-mon[66934]: Cluster is now healthy 2024-03-26T21:01:28.621 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:28 smithi063 ceph-mon[66934]: mgrmap e40: smithi063.lxhujt(active, since 2s), standbys: smithi174.sexoin 2024-03-26T21:01:28.621 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:28 smithi063 ceph-mon[66934]: [26/Mar/2024:21:01:27] ENGINE Serving on http://172.21.15.63:8765 2024-03-26T21:01:28.621 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:28 smithi063 ceph-mon[66934]: [26/Mar/2024:21:01:27] ENGINE Bus STARTED 2024-03-26T21:01:28.884 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:28 smithi174 ceph-mon[26934]: [26/Mar/2024:21:01:27] ENGINE Bus STARTING 2024-03-26T21:01:28.884 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:28 smithi174 ceph-mon[26934]: pgmap v4: 97 pgs: 97 active+clean; 345 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-03-26T21:01:28.884 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:28 smithi174 ceph-mon[26934]: [26/Mar/2024:21:01:27] ENGINE Serving on https://172.21.15.63:7150 2024-03-26T21:01:28.884 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:28 smithi174 ceph-mon[26934]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-03-26T21:01:28.884 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:28 smithi174 ceph-mon[26934]: Cluster is now healthy 2024-03-26T21:01:28.884 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:28 smithi174 ceph-mon[26934]: mgrmap e40: smithi063.lxhujt(active, since 2s), standbys: smithi174.sexoin 2024-03-26T21:01:28.885 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:28 smithi174 ceph-mon[26934]: [26/Mar/2024:21:01:27] ENGINE Serving on http://172.21.15.63:8765 2024-03-26T21:01:28.885 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:28 smithi174 ceph-mon[26934]: [26/Mar/2024:21:01:27] ENGINE Bus STARTED 2024-03-26T21:01:29.479 INFO:teuthology.orchestra.run.smithi063.stdout: 5 157880 11.58 MB/sec execute 203 sec latency 28.328 ms 2024-03-26T21:01:30.479 INFO:teuthology.orchestra.run.smithi063.stdout: 5 158289 11.60 MB/sec execute 204 sec latency 44.708 ms 2024-03-26T21:01:30.661 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:30 smithi063 ceph-mon[66934]: pgmap v5: 97 pgs: 97 active+clean; 345 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-03-26T21:01:30.662 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:30 smithi063 ceph-mon[66934]: mgrmap e41: smithi063.lxhujt(active, since 4s), standbys: smithi174.sexoin 2024-03-26T21:01:30.662 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:30 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:01:30.662 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:30 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:01:31.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:30 smithi174 ceph-mon[26934]: pgmap v5: 97 pgs: 97 active+clean; 345 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-03-26T21:01:31.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:30 smithi174 ceph-mon[26934]: mgrmap e41: smithi063.lxhujt(active, since 4s), standbys: smithi174.sexoin 2024-03-26T21:01:31.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:30 smithi174 ceph-mon[26934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:01:31.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:30 smithi174 ceph-mon[26934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:01:31.479 INFO:teuthology.orchestra.run.smithi063.stdout: 5 158840 11.63 MB/sec execute 205 sec latency 33.586 ms 2024-03-26T21:01:32.479 INFO:teuthology.orchestra.run.smithi063.stdout: 5 159328 11.64 MB/sec execute 206 sec latency 26.192 ms 2024-03-26T21:01:32.486 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:32 smithi174 ceph-mon[26934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:01:32.486 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:32 smithi174 ceph-mon[26934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:01:32.486 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:32 smithi174 ceph-mon[26934]: pgmap v6: 97 pgs: 97 active+clean; 345 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-03-26T21:01:32.673 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:32 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:01:32.674 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:32 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:01:32.674 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:32 smithi063 ceph-mon[66934]: pgmap v6: 97 pgs: 97 active+clean; 345 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-03-26T21:01:33.479 INFO:teuthology.orchestra.run.smithi063.stdout: 5 159674 11.62 MB/sec execute 207 sec latency 35.429 ms 2024-03-26T21:01:34.079 INFO:teuthology.orchestra.run.smithi063.stdout:true 2024-03-26T21:01:34.479 INFO:teuthology.orchestra.run.smithi063.stdout: 5 160108 11.62 MB/sec execute 208 sec latency 32.942 ms 2024-03-26T21:01:34.712 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:34 smithi063 ceph-mon[66934]: pgmap v7: 97 pgs: 97 active+clean; 345 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 36 KiB/s rd, 4.2 MiB/s wr, 456 op/s 2024-03-26T21:01:34.713 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:34 smithi063 ceph-mon[66934]: from='client.34132 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:01:34.946 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:34 smithi174 ceph-mon[26934]: pgmap v7: 97 pgs: 97 active+clean; 345 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 36 KiB/s rd, 4.2 MiB/s wr, 456 op/s 2024-03-26T21:01:34.946 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:34 smithi174 ceph-mon[26934]: from='client.34132 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:01:35.123 INFO:teuthology.orchestra.run.smithi063.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-26T21:01:35.123 INFO:teuthology.orchestra.run.smithi063.stdout:alertmanager.smithi063 smithi063 *:9093,9094 running (2m) 0s ago 11m 17.6M - 0.23.0 ba2b418f427c 6e160c94aac3 2024-03-26T21:01:35.123 INFO:teuthology.orchestra.run.smithi063.stdout:crash.smithi063 smithi063 running (11m) 0s ago 11m 6409k - 17.2.0 e1d6a67b021e 706c62f8af89 2024-03-26T21:01:35.123 INFO:teuthology.orchestra.run.smithi063.stdout:crash.smithi174 smithi174 running (11m) 4s ago 11m 6409k - 17.2.0 e1d6a67b021e 2c6bfe6d0ca6 2024-03-26T21:01:35.123 INFO:teuthology.orchestra.run.smithi063.stdout:grafana.smithi063 smithi063 *:3000 error 0s ago 11m - - 2024-03-26T21:01:35.123 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foofs.smithi063.tzaznz smithi063 running (6m) 0s ago 5m 37.2M - 17.2.0 e1d6a67b021e 10a989fc7954 2024-03-26T21:01:35.123 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foofs.smithi174.gitdvk smithi174 running (5m) 4s ago 5m 12.9M - 17.2.0 e1d6a67b021e 4339a5d98d1d 2024-03-26T21:01:35.123 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.smithi063.lxhujt smithi063 *:8443,9283,8765 running (106s) 0s ago 13m 477M - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 6d9d10330a90 2024-03-26T21:01:35.123 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.smithi174.sexoin smithi174 *:8443,9283,8765 running (51s) 4s ago 11m 448M - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 d7546233672d 2024-03-26T21:01:35.123 INFO:teuthology.orchestra.run.smithi063.stdout:mon.smithi063 smithi063 running (31s) 0s ago 13m 35.2M 2048M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 c644200344f8 2024-03-26T21:01:35.124 INFO:teuthology.orchestra.run.smithi063.stdout:mon.smithi174 smithi174 running (11m) 4s ago 11m 53.5M 2048M 17.2.0 e1d6a67b021e 0f81c09f0c0a 2024-03-26T21:01:35.124 INFO:teuthology.orchestra.run.smithi063.stdout:nfs.foo.0.0.smithi063.fgmpwa smithi063 *:2049 running (5m) 0s ago 5m 275M - 4.0 e1d6a67b021e 3d6c0dd9ce51 2024-03-26T21:01:35.124 INFO:teuthology.orchestra.run.smithi063.stdout:nfs.foo.1.0.smithi174.pytptt smithi174 *:2049 running (5m) 4s ago 5m 85.8M - 4.0 e1d6a67b021e f220dbb449a3 2024-03-26T21:01:35.124 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.smithi063 smithi063 *:9100 running (2m) 0s ago 11m 12.6M - 1.3.1 1dbe0e931976 f52f41fd14c0 2024-03-26T21:01:35.124 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.smithi174 smithi174 *:9100 running (116s) 4s ago 11m 17.2M - 1.3.1 1dbe0e931976 0c027f39a4c8 2024-03-26T21:01:35.124 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0 smithi063 running (10m) 0s ago 10m 564M 1936M 17.2.0 e1d6a67b021e 761b1388d8fa 2024-03-26T21:01:35.124 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1 smithi063 running (9m) 0s ago 9m 481M 1936M 17.2.0 e1d6a67b021e 108275c17cad 2024-03-26T21:01:35.124 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2 smithi063 running (9m) 0s ago 9m 334M 1936M 17.2.0 e1d6a67b021e 107f685ad08d 2024-03-26T21:01:35.124 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3 smithi063 running (9m) 0s ago 9m 500M 1936M 17.2.0 e1d6a67b021e ab0f1a855957 2024-03-26T21:01:35.124 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4 smithi174 running (8m) 4s ago 8m 522M 2704M 17.2.0 e1d6a67b021e a0e877ecc06d 2024-03-26T21:01:35.124 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5 smithi174 running (8m) 4s ago 8m 328M 2704M 17.2.0 e1d6a67b021e 378c2aca7e71 2024-03-26T21:01:35.124 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6 smithi174 running (7m) 4s ago 7m 510M 2704M 17.2.0 e1d6a67b021e ffbb5b7ad3b6 2024-03-26T21:01:35.124 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7 smithi174 running (7m) 4s ago 7m 393M 2704M 17.2.0 e1d6a67b021e c1d57fcabb3f 2024-03-26T21:01:35.124 INFO:teuthology.orchestra.run.smithi063.stdout:prometheus.smithi063 smithi063 *:9095 running (57s) 0s ago 10m 62.0M - 2.33.4 514e6a882f6e 567eef202627 2024-03-26T21:01:35.479 INFO:teuthology.orchestra.run.smithi063.stdout: 5 160395 11.61 MB/sec execute 209 sec latency 61.628 ms 2024-03-26T21:01:35.732 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:35 smithi063 ceph-mon[66934]: from='client.34136 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:01:35.732 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:35 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:01:35.732 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:35 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:01:35.733 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:35 smithi063 ceph-mon[66934]: from='client.34140 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:01:35.779 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-03-26T21:01:35.780 INFO:teuthology.orchestra.run.smithi063.stdout: "mon": { 2024-03-26T21:01:35.780 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-03-26T21:01:35.780 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 1 2024-03-26T21:01:35.780 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:01:35.780 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr": { 2024-03-26T21:01:35.780 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 2 2024-03-26T21:01:35.780 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:01:35.780 INFO:teuthology.orchestra.run.smithi063.stdout: "osd": { 2024-03-26T21:01:35.780 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-03-26T21:01:35.780 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:01:35.780 INFO:teuthology.orchestra.run.smithi063.stdout: "mds": { 2024-03-26T21:01:35.780 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-26T21:01:35.780 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:01:35.781 INFO:teuthology.orchestra.run.smithi063.stdout: "overall": { 2024-03-26T21:01:35.781 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 11, 2024-03-26T21:01:35.781 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 3 2024-03-26T21:01:35.781 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-03-26T21:01:35.781 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-03-26T21:01:36.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:35 smithi174 ceph-mon[26934]: from='client.34136 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:01:36.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:35 smithi174 ceph-mon[26934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:01:36.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:35 smithi174 ceph-mon[26934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:01:36.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:35 smithi174 ceph-mon[26934]: from='client.34140 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:01:36.339 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-03-26T21:01:36.339 INFO:teuthology.orchestra.run.smithi063.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d", 2024-03-26T21:01:36.339 INFO:teuthology.orchestra.run.smithi063.stdout: "in_progress": true, 2024-03-26T21:01:36.339 INFO:teuthology.orchestra.run.smithi063.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-26T21:01:36.340 INFO:teuthology.orchestra.run.smithi063.stdout: "services_complete": [ 2024-03-26T21:01:36.340 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr" 2024-03-26T21:01:36.340 INFO:teuthology.orchestra.run.smithi063.stdout: ], 2024-03-26T21:01:36.340 INFO:teuthology.orchestra.run.smithi063.stdout: "progress": "3/23 daemons upgraded", 2024-03-26T21:01:36.340 INFO:teuthology.orchestra.run.smithi063.stdout: "message": "", 2024-03-26T21:01:36.340 INFO:teuthology.orchestra.run.smithi063.stdout: "is_paused": false 2024-03-26T21:01:36.340 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-03-26T21:01:36.479 INFO:teuthology.orchestra.run.smithi063.stdout: 5 160739 11.61 MB/sec execute 210 sec latency 63.952 ms 2024-03-26T21:01:36.745 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:36 smithi063 ceph-mon[66934]: pgmap v8: 97 pgs: 97 active+clean; 345 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 28 KiB/s rd, 3.2 MiB/s wr, 353 op/s 2024-03-26T21:01:36.745 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:36 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:01:36.745 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:36 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:01:36.745 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:36 smithi063 ceph-mon[66934]: from='client.? 172.21.15.63:0/3375842538' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:01:36.745 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:36 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:01:36.745 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:36 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:01:36.745 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:36 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:01:36.746 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:36 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T21:01:37.038 INFO:teuthology.orchestra.run.smithi063.stdout:HEALTH_OK 2024-03-26T21:01:37.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:36 smithi174 ceph-mon[26934]: pgmap v8: 97 pgs: 97 active+clean; 345 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 28 KiB/s rd, 3.2 MiB/s wr, 353 op/s 2024-03-26T21:01:37.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:36 smithi174 ceph-mon[26934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:01:37.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:36 smithi174 ceph-mon[26934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:01:37.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:36 smithi174 ceph-mon[26934]: from='client.? 172.21.15.63:0/3375842538' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:01:37.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:36 smithi174 ceph-mon[26934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:01:37.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:36 smithi174 ceph-mon[26934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:01:37.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:36 smithi174 ceph-mon[26934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:01:37.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:36 smithi174 ceph-mon[26934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T21:01:37.479 INFO:teuthology.orchestra.run.smithi063.stdout: 5 161077 11.57 MB/sec execute 211 sec latency 70.578 ms 2024-03-26T21:01:37.758 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:37 smithi063 ceph-mon[66934]: Updating smithi063:/etc/ceph/ceph.conf 2024-03-26T21:01:37.758 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:37 smithi063 ceph-mon[66934]: Updating smithi174:/etc/ceph/ceph.conf 2024-03-26T21:01:37.758 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:37 smithi063 ceph-mon[66934]: from='client.24665 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:01:37.758 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:37 smithi063 ceph-mon[66934]: Updating smithi174:/var/lib/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-26T21:01:37.759 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:37 smithi063 ceph-mon[66934]: Updating smithi063:/var/lib/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-26T21:01:37.759 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:37 smithi063 ceph-mon[66934]: Updating smithi174:/etc/ceph/ceph.client.admin.keyring 2024-03-26T21:01:37.759 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:37 smithi063 ceph-mon[66934]: Updating smithi063:/etc/ceph/ceph.client.admin.keyring 2024-03-26T21:01:37.759 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:37 smithi063 ceph-mon[66934]: from='client.? 172.21.15.63:0/956349924' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-26T21:01:37.759 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:37 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:01:37.759 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:37 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:01:37.760 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:37 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:01:37.760 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:37 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:01:37.760 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:37 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:01:37.760 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:37 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:01:38.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:37 smithi174 ceph-mon[26934]: Updating smithi063:/etc/ceph/ceph.conf 2024-03-26T21:01:38.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:37 smithi174 ceph-mon[26934]: Updating smithi174:/etc/ceph/ceph.conf 2024-03-26T21:01:38.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:37 smithi174 ceph-mon[26934]: from='client.24665 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:01:38.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:37 smithi174 ceph-mon[26934]: Updating smithi174:/var/lib/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-26T21:01:38.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:37 smithi174 ceph-mon[26934]: Updating smithi063:/var/lib/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-26T21:01:38.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:37 smithi174 ceph-mon[26934]: Updating smithi174:/etc/ceph/ceph.client.admin.keyring 2024-03-26T21:01:38.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:37 smithi174 ceph-mon[26934]: Updating smithi063:/etc/ceph/ceph.client.admin.keyring 2024-03-26T21:01:38.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:37 smithi174 ceph-mon[26934]: from='client.? 172.21.15.63:0/956349924' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-26T21:01:38.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:37 smithi174 ceph-mon[26934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:01:38.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:37 smithi174 ceph-mon[26934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:01:38.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:37 smithi174 ceph-mon[26934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:01:38.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:37 smithi174 ceph-mon[26934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:01:38.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:37 smithi174 ceph-mon[26934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:01:38.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:37 smithi174 ceph-mon[26934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:01:38.480 INFO:teuthology.orchestra.run.smithi063.stdout: 5 161414 11.54 MB/sec execute 212 sec latency 53.496 ms 2024-03-26T21:01:38.780 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:38 smithi063 ceph-mon[66934]: Updating smithi174:/var/lib/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-26T21:01:38.780 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:38 smithi063 ceph-mon[66934]: Updating smithi063:/var/lib/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-26T21:01:38.780 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:38 smithi063 ceph-mon[66934]: pgmap v9: 97 pgs: 97 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 27 KiB/s rd, 5.3 MiB/s wr, 546 op/s 2024-03-26T21:01:38.780 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:38 smithi063 ceph-mon[66934]: pgmap v10: 97 pgs: 97 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 29 KiB/s rd, 5.6 MiB/s wr, 587 op/s 2024-03-26T21:01:38.780 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:38 smithi063 ceph-mon[66934]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-03-26T21:01:38.780 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:38 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:01:38.780 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:38 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:01:38.780 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:38 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-03-26T21:01:39.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:38 smithi174 ceph-mon[26934]: Updating smithi174:/var/lib/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-26T21:01:39.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:38 smithi174 ceph-mon[26934]: Updating smithi063:/var/lib/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-26T21:01:39.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:38 smithi174 ceph-mon[26934]: pgmap v9: 97 pgs: 97 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 27 KiB/s rd, 5.3 MiB/s wr, 546 op/s 2024-03-26T21:01:39.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:38 smithi174 ceph-mon[26934]: pgmap v10: 97 pgs: 97 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 29 KiB/s rd, 5.6 MiB/s wr, 587 op/s 2024-03-26T21:01:39.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:38 smithi174 ceph-mon[26934]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-03-26T21:01:39.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:38 smithi174 ceph-mon[26934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:01:39.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:38 smithi174 ceph-mon[26934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:01:39.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:38 smithi174 ceph-mon[26934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-03-26T21:01:39.479 INFO:teuthology.orchestra.run.smithi063.stdout: 5 161771 11.54 MB/sec execute 213 sec latency 48.075 ms 2024-03-26T21:01:40.479 INFO:teuthology.orchestra.run.smithi063.stdout: 5 162131 11.55 MB/sec execute 214 sec latency 55.570 ms 2024-03-26T21:01:40.825 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:40 smithi063 ceph-mon[66934]: Upgrade: Updating mon.smithi174 2024-03-26T21:01:40.826 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:40 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:01:40.826 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:40 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-26T21:01:40.826 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:40 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-26T21:01:40.826 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:40 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:01:40.826 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:40 smithi063 ceph-mon[66934]: Deploying daemon mon.smithi174 on smithi174 2024-03-26T21:01:40.826 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:40 smithi063 ceph-mon[66934]: pgmap v11: 97 pgs: 97 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 29 KiB/s rd, 5.6 MiB/s wr, 587 op/s 2024-03-26T21:01:41.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:40 smithi174 ceph-mon[26934]: Upgrade: Updating mon.smithi174 2024-03-26T21:01:41.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:40 smithi174 ceph-mon[26934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:01:41.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:40 smithi174 ceph-mon[26934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-26T21:01:41.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:40 smithi174 ceph-mon[26934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-26T21:01:41.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:40 smithi174 ceph-mon[26934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:01:41.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:40 smithi174 ceph-mon[26934]: Deploying daemon mon.smithi174 on smithi174 2024-03-26T21:01:41.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:40 smithi174 ceph-mon[26934]: pgmap v11: 97 pgs: 97 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 29 KiB/s rd, 5.6 MiB/s wr, 587 op/s 2024-03-26T21:01:41.479 INFO:teuthology.orchestra.run.smithi063.stdout: 5 162554 11.56 MB/sec execute 215 sec latency 55.573 ms 2024-03-26T21:01:41.599 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:41 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:01:41.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:41 smithi174 ceph-mon[26934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:01:42.109 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:41 smithi174 systemd[1]: Stopping Ceph mon.smithi174 for 0ed4dd4c-ebb2-11ee-95d0-87774f69a715... 2024-03-26T21:01:42.434 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:42 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-mon-smithi174[26914]: 2024-03-26T21:01:42.114+0000 7fa08aae0700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi174 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-03-26T21:01:42.434 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:42 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-mon-smithi174[26914]: 2024-03-26T21:01:42.114+0000 7fa08aae0700 -1 mon.smithi174@1(peon) e2 *** Got Signal Terminated *** 2024-03-26T21:01:42.435 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:42 smithi174 podman[52857]: 2024-03-26 21:01:42.135376697 +0000 UTC m=+0.136068160 container died 0f81c09f0c0a313fb043f66995f4c10b82c287f81f2407a14db6b57d0835f322 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-mon-smithi174, name=centos-stream, version=8, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, vcs-type=git, release=754, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, io.openshift.expose-services=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-03-26T21:01:42.479 INFO:teuthology.orchestra.run.smithi063.stdout: 5 162857 11.54 MB/sec execute 216 sec latency 56.723 ms 2024-03-26T21:01:42.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:42 smithi174 podman[52857]: 2024-03-26 21:01:42.54008859 +0000 UTC m=+0.540780048 container cleanup 0f81c09f0c0a313fb043f66995f4c10b82c287f81f2407a14db6b57d0835f322 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-mon-smithi174, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, GIT_CLEAN=True, version=8, io.buildah.version=1.19.8, architecture=x86_64, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, name=centos-stream, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-03-26T21:01:42.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:42 smithi174 bash[52857]: ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-mon-smithi174 2024-03-26T21:01:43.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 podman[52871]: 2024-03-26 21:01:43.053135504 +0000 UTC m=+0.917620030 container remove 0f81c09f0c0a313fb043f66995f4c10b82c287f81f2407a14db6b57d0835f322 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-mon-smithi174, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, vendor=Red Hat, Inc., ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , GIT_CLEAN=True, architecture=x86_64, version=8, io.openshift.expose-services=, GIT_BRANCH=HEAD) 2024-03-26T21:01:43.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 systemd[1]: ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715@mon.smithi174.service: Deactivated successfully. 2024-03-26T21:01:43.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 systemd[1]: Stopped Ceph mon.smithi174 for 0ed4dd4c-ebb2-11ee-95d0-87774f69a715. 2024-03-26T21:01:43.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 systemd[1]: ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715@mon.smithi174.service: Consumed 7.334s CPU time. 2024-03-26T21:01:43.483 INFO:teuthology.orchestra.run.smithi063.stdout: 5 163046 11.50 MB/sec execute 217 sec latency 36.400 ms 2024-03-26T21:01:43.686 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 systemd[1]: Starting Ceph mon.smithi174 for 0ed4dd4c-ebb2-11ee-95d0-87774f69a715... 2024-03-26T21:01:43.686 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 podman[52996]: 2024-03-26 21:01:43.563891631 +0000 UTC m=+0.023229470 image pull a9bdc3eaeb106993bcbbac35d7fe5c7b76c052021676e129f5984aed30aee063 quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d 2024-03-26T21:01:43.686 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 podman[52996]: 2024-03-26 21:01:43.693115429 +0000 UTC m=+0.152453265 container create 1a6d442692e789ee427d8d48fa89d6faa44467ae591f6c2107e29d458209b5ae (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-mon-smithi174, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240326, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=) 2024-03-26T21:01:43.964 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 podman[52996]: 2024-03-26 21:01:43.93800727 +0000 UTC m=+0.397345118 container init 1a6d442692e789ee427d8d48fa89d6faa44467ae591f6c2107e29d458209b5ae (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-mon-smithi174, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240326, maintainer=Guillaume Abrioux , RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5) 2024-03-26T21:01:43.964 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 podman[52996]: 2024-03-26 21:01:43.94315486 +0000 UTC m=+0.402492701 container start 1a6d442692e789ee427d8d48fa89d6faa44467ae591f6c2107e29d458209b5ae (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-mon-smithi174, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, GIT_CLEAN=True, ceph=True) 2024-03-26T21:01:44.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: set uid:gid to 167:167 (ceph:ceph) 2024-03-26T21:01:44.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev), process ceph-mon, pid 2 2024-03-26T21:01:44.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: pidfile_write: ignore empty --pid-file 2024-03-26T21:01:44.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: load: jerasure load: lrc 2024-03-26T21:01:44.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: RocksDB version: 7.9.2 2024-03-26T21:01:44.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Git sha 0 2024-03-26T21:01:44.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Compile date 2024-03-26 11:42:14 2024-03-26T21:01:44.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: DB SUMMARY 2024-03-26T21:01:44.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: DB Session ID: UDBBGV78O65XUZM7O77P 2024-03-26T21:01:44.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: CURRENT file: CURRENT 2024-03-26T21:01:44.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: IDENTITY file: IDENTITY 2024-03-26T21:01:44.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: MANIFEST file: MANIFEST-000009 size: 903 Bytes 2024-03-26T21:01:44.300 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi174/store.db dir, Total Num: 1, files: 000024.sst 2024-03-26T21:01:44.300 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi174/store.db: 000022.log size: 7933224 ; 2024-03-26T21:01:44.300 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.error_if_exists: 0 2024-03-26T21:01:44.300 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.create_if_missing: 0 2024-03-26T21:01:44.300 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.paranoid_checks: 1 2024-03-26T21:01:44.300 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.flush_verify_memtable_count: 1 2024-03-26T21:01:44.300 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-03-26T21:01:44.300 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-03-26T21:01:44.300 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.env: 0x562f01cd4c60 2024-03-26T21:01:44.300 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.fs: PosixFileSystem 2024-03-26T21:01:44.300 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.info_log: 0x562f028319a0 2024-03-26T21:01:44.300 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.max_file_opening_threads: 16 2024-03-26T21:01:44.300 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.statistics: (nil) 2024-03-26T21:01:44.300 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.use_fsync: 0 2024-03-26T21:01:44.301 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.max_log_file_size: 0 2024-03-26T21:01:44.301 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-03-26T21:01:44.301 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.log_file_time_to_roll: 0 2024-03-26T21:01:44.301 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.keep_log_file_num: 1000 2024-03-26T21:01:44.301 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.recycle_log_file_num: 0 2024-03-26T21:01:44.301 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.allow_fallocate: 1 2024-03-26T21:01:44.301 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.allow_mmap_reads: 0 2024-03-26T21:01:44.301 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.allow_mmap_writes: 0 2024-03-26T21:01:44.301 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.use_direct_reads: 0 2024-03-26T21:01:44.301 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-03-26T21:01:44.301 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.create_missing_column_families: 0 2024-03-26T21:01:44.301 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.db_log_dir: 2024-03-26T21:01:44.301 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.wal_dir: 2024-03-26T21:01:44.301 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.table_cache_numshardbits: 6 2024-03-26T21:01:44.303 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.WAL_ttl_seconds: 0 2024-03-26T21:01:44.303 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.WAL_size_limit_MB: 0 2024-03-26T21:01:44.303 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-03-26T21:01:44.303 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-03-26T21:01:44.303 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.is_fd_close_on_exec: 1 2024-03-26T21:01:44.303 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.advise_random_on_open: 1 2024-03-26T21:01:44.303 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.db_write_buffer_size: 0 2024-03-26T21:01:44.303 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.write_buffer_manager: 0x562f028021e0 2024-03-26T21:01:44.303 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-03-26T21:01:44.304 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-03-26T21:01:44.304 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.use_adaptive_mutex: 0 2024-03-26T21:01:44.304 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.rate_limiter: (nil) 2024-03-26T21:01:44.304 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-03-26T21:01:44.304 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.wal_recovery_mode: 2 2024-03-26T21:01:44.304 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.enable_thread_tracking: 0 2024-03-26T21:01:44.304 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.enable_pipelined_write: 0 2024-03-26T21:01:44.304 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.unordered_write: 0 2024-03-26T21:01:44.304 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-03-26T21:01:44.304 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-03-26T21:01:44.304 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-03-26T21:01:44.304 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-03-26T21:01:44.304 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.row_cache: None 2024-03-26T21:01:44.304 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.wal_filter: None 2024-03-26T21:01:44.304 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-03-26T21:01:44.305 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.allow_ingest_behind: 0 2024-03-26T21:01:44.305 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.two_write_queues: 0 2024-03-26T21:01:44.305 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.manual_wal_flush: 0 2024-03-26T21:01:44.305 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.wal_compression: 0 2024-03-26T21:01:44.305 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.atomic_flush: 0 2024-03-26T21:01:44.305 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-03-26T21:01:44.305 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.persist_stats_to_disk: 0 2024-03-26T21:01:44.305 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.write_dbid_to_manifest: 0 2024-03-26T21:01:44.305 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.log_readahead_size: 0 2024-03-26T21:01:44.305 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-03-26T21:01:44.305 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.best_efforts_recovery: 0 2024-03-26T21:01:44.305 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-03-26T21:01:44.305 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-03-26T21:01:44.305 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.allow_data_in_errors: 0 2024-03-26T21:01:44.305 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.db_host_id: __hostname__ 2024-03-26T21:01:44.306 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.enforce_single_del_contracts: true 2024-03-26T21:01:44.306 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.max_background_jobs: 2 2024-03-26T21:01:44.306 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.max_background_compactions: -1 2024-03-26T21:01:44.306 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.max_subcompactions: 1 2024-03-26T21:01:44.306 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-03-26T21:01:44.306 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-03-26T21:01:44.306 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.delayed_write_rate : 16777216 2024-03-26T21:01:44.306 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.max_total_wal_size: 0 2024-03-26T21:01:44.306 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-03-26T21:01:44.306 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.stats_dump_period_sec: 600 2024-03-26T21:01:44.306 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.stats_persist_period_sec: 600 2024-03-26T21:01:44.306 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-03-26T21:01:44.306 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.max_open_files: -1 2024-03-26T21:01:44.306 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.bytes_per_sync: 0 2024-03-26T21:01:44.306 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.wal_bytes_per_sync: 0 2024-03-26T21:01:44.307 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.strict_bytes_per_sync: 0 2024-03-26T21:01:44.307 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.compaction_readahead_size: 0 2024-03-26T21:01:44.307 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.max_background_flushes: -1 2024-03-26T21:01:44.307 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Compression algorithms supported: 2024-03-26T21:01:44.307 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-03-26T21:01:44.307 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: kZSTD supported: 0 2024-03-26T21:01:44.307 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: kXpressCompression supported: 0 2024-03-26T21:01:44.307 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: kLZ4HCCompression supported: 1 2024-03-26T21:01:44.307 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: kZlibCompression supported: 1 2024-03-26T21:01:44.307 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: kSnappyCompression supported: 1 2024-03-26T21:01:44.307 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: kLZ4Compression supported: 1 2024-03-26T21:01:44.307 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: kBZip2Compression supported: 0 2024-03-26T21:01:44.307 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Fast CRC32 supported: Supported on x86 2024-03-26T21:01:44.307 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: DMutex implementation: pthread_mutex_t 2024-03-26T21:01:44.307 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi174/store.db/MANIFEST-000009 2024-03-26T21:01:44.308 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-03-26T21:01:44.308 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-03-26T21:01:44.308 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.merge_operator: 2024-03-26T21:01:44.308 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.compaction_filter: None 2024-03-26T21:01:44.308 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.compaction_filter_factory: None 2024-03-26T21:01:44.308 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.sst_partitioner_factory: None 2024-03-26T21:01:44.308 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.memtable_factory: SkipListFactory 2024-03-26T21:01:44.308 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.table_factory: BlockBasedTable 2024-03-26T21:01:44.308 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x562f02831b60) 2024-03-26T21:01:44.308 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: cache_index_and_filter_blocks: 1 2024-03-26T21:01:44.308 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-03-26T21:01:44.308 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-03-26T21:01:44.308 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: pin_top_level_index_and_filter: 1 2024-03-26T21:01:44.308 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: index_type: 0 2024-03-26T21:01:44.309 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: data_block_index_type: 0 2024-03-26T21:01:44.309 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: index_shortening: 1 2024-03-26T21:01:44.309 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: data_block_hash_table_util_ratio: 0.750000 2024-03-26T21:01:44.309 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: checksum: 4 2024-03-26T21:01:44.309 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: no_block_cache: 0 2024-03-26T21:01:44.309 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: block_cache: 0x562f02835090 2024-03-26T21:01:44.309 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: block_cache_name: BinnedLRUCache 2024-03-26T21:01:44.309 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: block_cache_options: 2024-03-26T21:01:44.309 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: capacity : 536870912 2024-03-26T21:01:44.309 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: num_shard_bits : 4 2024-03-26T21:01:44.309 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: strict_capacity_limit : 0 2024-03-26T21:01:44.309 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: high_pri_pool_ratio: 0.000 2024-03-26T21:01:44.309 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: block_cache_compressed: (nil) 2024-03-26T21:01:44.309 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: persistent_cache: (nil) 2024-03-26T21:01:44.309 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: block_size: 4096 2024-03-26T21:01:44.310 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: block_size_deviation: 10 2024-03-26T21:01:44.310 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: block_restart_interval: 16 2024-03-26T21:01:44.310 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: index_block_restart_interval: 1 2024-03-26T21:01:44.310 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: metadata_block_size: 4096 2024-03-26T21:01:44.310 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: partition_filters: 0 2024-03-26T21:01:44.310 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: use_delta_encoding: 1 2024-03-26T21:01:44.310 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: filter_policy: bloomfilter 2024-03-26T21:01:44.310 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: whole_key_filtering: 1 2024-03-26T21:01:44.310 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: verify_compression: 0 2024-03-26T21:01:44.310 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: read_amp_bytes_per_bit: 0 2024-03-26T21:01:44.310 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: format_version: 5 2024-03-26T21:01:44.310 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: enable_index_compression: 1 2024-03-26T21:01:44.310 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: block_align: 0 2024-03-26T21:01:44.310 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: max_auto_readahead_size: 262144 2024-03-26T21:01:44.311 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: prepopulate_block_cache: 0 2024-03-26T21:01:44.311 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: initial_auto_readahead_size: 8192 2024-03-26T21:01:44.311 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: num_file_reads_for_auto_readahead: 2 2024-03-26T21:01:44.311 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.write_buffer_size: 33554432 2024-03-26T21:01:44.311 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.max_write_buffer_number: 2 2024-03-26T21:01:44.311 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.compression: NoCompression 2024-03-26T21:01:44.311 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.bottommost_compression: Disabled 2024-03-26T21:01:44.311 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.prefix_extractor: nullptr 2024-03-26T21:01:44.311 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-03-26T21:01:44.311 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.num_levels: 7 2024-03-26T21:01:44.311 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-03-26T21:01:44.311 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-03-26T21:01:44.311 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-03-26T21:01:44.311 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-03-26T21:01:44.311 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-03-26T21:01:44.312 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-03-26T21:01:44.312 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-03-26T21:01:44.312 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-03-26T21:01:44.312 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-03-26T21:01:44.312 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-03-26T21:01:44.312 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-03-26T21:01:44.312 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-03-26T21:01:44.312 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.compression_opts.window_bits: -14 2024-03-26T21:01:44.312 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.compression_opts.level: 32767 2024-03-26T21:01:44.312 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.compression_opts.strategy: 0 2024-03-26T21:01:44.312 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-03-26T21:01:44.312 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-03-26T21:01:44.313 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-03-26T21:01:44.313 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-03-26T21:01:44.313 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.compression_opts.enabled: false 2024-03-26T21:01:44.313 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-03-26T21:01:44.313 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-03-26T21:01:44.313 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-03-26T21:01:44.313 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-03-26T21:01:44.313 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.target_file_size_base: 67108864 2024-03-26T21:01:44.313 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.target_file_size_multiplier: 1 2024-03-26T21:01:44.313 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-03-26T21:01:44.313 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-03-26T21:01:44.313 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-03-26T21:01:44.313 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-03-26T21:01:44.313 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-03-26T21:01:44.313 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-03-26T21:01:44.314 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-03-26T21:01:44.314 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-03-26T21:01:44.314 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-03-26T21:01:44.314 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-03-26T21:01:44.314 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-03-26T21:01:44.314 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-03-26T21:01:44.314 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-03-26T21:01:44.314 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.arena_block_size: 1048576 2024-03-26T21:01:44.314 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-03-26T21:01:44.314 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-03-26T21:01:44.314 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.disable_auto_compactions: 0 2024-03-26T21:01:44.314 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-03-26T21:01:44.314 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-03-26T21:01:44.314 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-03-26T21:01:44.315 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-03-26T21:01:44.315 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-03-26T21:01:44.315 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-03-26T21:01:44.315 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-03-26T21:01:44.315 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-03-26T21:01:44.315 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-03-26T21:01:44.315 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-03-26T21:01:44.315 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-03-26T21:01:44.315 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.inplace_update_support: 0 2024-03-26T21:01:44.315 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.inplace_update_num_locks: 10000 2024-03-26T21:01:44.315 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-03-26T21:01:44.315 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-03-26T21:01:44.315 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.memtable_huge_page_size: 0 2024-03-26T21:01:44.315 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.bloom_locality: 0 2024-03-26T21:01:44.315 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.max_successive_merges: 0 2024-03-26T21:01:44.316 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.optimize_filters_for_hits: 0 2024-03-26T21:01:44.316 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.paranoid_file_checks: 0 2024-03-26T21:01:44.316 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.force_consistency_checks: 1 2024-03-26T21:01:44.316 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.report_bg_io_stats: 0 2024-03-26T21:01:44.316 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.ttl: 2592000 2024-03-26T21:01:44.316 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.periodic_compaction_seconds: 0 2024-03-26T21:01:44.316 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-03-26T21:01:44.316 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-03-26T21:01:44.316 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.enable_blob_files: false 2024-03-26T21:01:44.316 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.min_blob_size: 0 2024-03-26T21:01:44.316 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.blob_file_size: 268435456 2024-03-26T21:01:44.316 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.blob_compression_type: NoCompression 2024-03-26T21:01:44.316 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.enable_blob_garbage_collection: false 2024-03-26T21:01:44.316 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-03-26T21:01:44.316 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-03-26T21:01:44.317 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-03-26T21:01:44.317 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.blob_file_starting_level: 0 2024-03-26T21:01:44.317 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-03-26T21:01:44.317 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 24.sst 2024-03-26T21:01:44.317 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-03-26T21:01:44.317 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi174/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 26, last_sequence is 12166, log_number is 22,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-03-26T21:01:44.317 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 22 2024-03-26T21:01:44.317 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: e03e2987-d271-4a52-ac34-2ad41c0c05f0 2024-03-26T21:01:44.317 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: EVENT_LOG_v1 {"time_micros": 1711486903977929, "job": 1, "event": "recovery_started", "wal_files": [22]} 2024-03-26T21:01:44.317 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #22 mode 2 2024-03-26T21:01:44.317 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:43 smithi174 ceph-mon[53014]: rocksdb: EVENT_LOG_v1 {"time_micros": 1711486903999879, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 27, "file_size": 4844829, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12171, "largest_seqno": 14084, "table_properties": {"data_size": 4836417, "index_size": 5446, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1925, "raw_key_size": 17533, "raw_average_key_size": 23, "raw_value_size": 4820011, "raw_average_value_size": 6350, "num_data_blocks": 246, "num_entries": 759, "num_filter_entries": 759, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1711486903, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "e03e2987-d271-4a52-ac34-2ad41c0c05f0", "db_session_id": "UDBBGV78O65XUZM7O77P", "orig_file_number": 27, "seqno_to_time_mapping": "N/A"}} 2024-03-26T21:01:44.317 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:44 smithi174 ceph-mon[53014]: rocksdb: EVENT_LOG_v1 {"time_micros": 1711486904000000, "job": 1, "event": "recovery_finished"} 2024-03-26T21:01:44.317 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:44 smithi174 ceph-mon[53014]: rocksdb: [db/version_set.cc:5047] Creating manifest 29 2024-03-26T21:01:44.317 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:44 smithi174 ceph-mon[53014]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-03-26T21:01:44.317 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:44 smithi174 ceph-mon[53014]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-smithi174/store.db/000022.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-03-26T21:01:44.318 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:44 smithi174 ceph-mon[53014]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x562f02924000 2024-03-26T21:01:44.318 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:44 smithi174 ceph-mon[53014]: rocksdb: DB pointer 0x562f0290e000 2024-03-26T21:01:44.318 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:44 smithi174 ceph-mon[53014]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-03-26T21:01:44.318 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:44 smithi174 ceph-mon[53014]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-03-26T21:01:44.318 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: ** DB Stats ** 2024-03-26T21:01:44.318 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-03-26T21:01:44.318 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-03-26T21:01:44.318 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-03-26T21:01:44.318 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-26T21:01:44.318 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-03-26T21:01:44.318 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-03-26T21:01:44.318 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-26T21:01:44.318 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: 2024-03-26T21:01:44.319 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: ** Compaction Stats [default] ** 2024-03-26T21:01:44.319 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-03-26T21:01:44.319 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-03-26T21:01:44.319 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: L0 1/0 4.62 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 498.7 0.01 0.00 1 0.009 0 0 0.0 0.0 2024-03-26T21:01:44.319 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: L6 1/0 6.20 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-03-26T21:01:44.319 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Sum 2/0 10.82 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 498.7 0.01 0.00 1 0.009 0 0 0.0 0.0 2024-03-26T21:01:44.319 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 498.7 0.01 0.00 1 0.009 0 0 0.0 0.0 2024-03-26T21:01:44.319 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: 2024-03-26T21:01:44.319 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: ** Compaction Stats [default] ** 2024-03-26T21:01:44.319 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-03-26T21:01:44.319 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-03-26T21:01:44.319 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 498.7 0.01 0.00 1 0.009 0 0 0.0 0.0 2024-03-26T21:01:44.319 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: 2024-03-26T21:01:44.319 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-03-26T21:01:44.320 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: 2024-03-26T21:01:44.320 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-03-26T21:01:44.320 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Flush(GB): cumulative 0.005, interval 0.005 2024-03-26T21:01:44.320 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-03-26T21:01:44.320 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-03-26T21:01:44.320 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-03-26T21:01:44.320 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: AddFile(Keys): cumulative 0, interval 0 2024-03-26T21:01:44.320 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Cumulative compaction: 0.00 GB write, 160.13 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-26T21:01:44.320 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Interval compaction: 0.00 GB write, 160.13 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-26T21:01:44.320 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-03-26T21:01:44.320 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Block cache BinnedLRUCache@0x562f02835090#2 capacity: 512.00 MB usage: 8.11 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.7e-05 secs_since: 0 2024-03-26T21:01:44.320 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,2.03 KB,0.00038743%) IndexBlock(1,6.08 KB,0.00115931%) Misc(1,0.00 KB,0%) 2024-03-26T21:01:44.320 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: 2024-03-26T21:01:44.320 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: ** File Read Latency Histogram By Level [default] ** 2024-03-26T21:01:44.321 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:44 smithi174 ceph-mon[53014]: starting mon.smithi174 rank 1 at public addrs [v2:172.21.15.174:3300/0,v1:172.21.15.174:6789/0] at bind addrs [v2:172.21.15.174:3300/0,v1:172.21.15.174:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi174 fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 2024-03-26T21:01:44.321 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:44 smithi174 ceph-mon[53014]: mon.smithi174@-1(???) e2 preinit fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 2024-03-26T21:01:44.321 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:44 smithi174 ceph-mon[53014]: mon.smithi174@-1(???).mds e7 new map 2024-03-26T21:01:44.321 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:44 smithi174 ceph-mon[53014]: mon.smithi174@-1(???).mds e7 print_map 2024-03-26T21:01:44.321 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: e7 2024-03-26T21:01:44.321 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-03-26T21:01:44.321 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-03-26T21:01:44.321 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: legacy client fscid: 1 2024-03-26T21:01:44.321 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: 2024-03-26T21:01:44.321 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Filesystem 'foofs' (1) 2024-03-26T21:01:44.321 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: fs_name foofs 2024-03-26T21:01:44.321 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: epoch 5 2024-03-26T21:01:44.321 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-03-26T21:01:44.321 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: created 2024-03-26T20:55:32.397790+0000 2024-03-26T21:01:44.321 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: modified 2024-03-26T20:55:36.411042+0000 2024-03-26T21:01:44.322 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: tableserver 0 2024-03-26T21:01:44.322 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: root 0 2024-03-26T21:01:44.322 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: session_timeout 60 2024-03-26T21:01:44.322 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: session_autoclose 300 2024-03-26T21:01:44.322 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: max_file_size 1099511627776 2024-03-26T21:01:44.322 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: max_xattr_size 65536 2024-03-26T21:01:44.322 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: required_client_features {} 2024-03-26T21:01:44.322 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: last_failure 0 2024-03-26T21:01:44.322 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: last_failure_osd_epoch 0 2024-03-26T21:01:44.322 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-03-26T21:01:44.322 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: max_mds 1 2024-03-26T21:01:44.322 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: in 0 2024-03-26T21:01:44.322 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: up {0=14474} 2024-03-26T21:01:44.322 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: failed 2024-03-26T21:01:44.323 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: damaged 2024-03-26T21:01:44.323 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: stopped 2024-03-26T21:01:44.323 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: data_pools [3] 2024-03-26T21:01:44.323 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: metadata_pool 2 2024-03-26T21:01:44.323 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: inline_data disabled 2024-03-26T21:01:44.323 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: balancer 2024-03-26T21:01:44.323 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: bal_rank_mask -1 2024-03-26T21:01:44.323 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: standby_count_wanted 1 2024-03-26T21:01:44.323 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: qdb_cluster leader: 0 members: 2024-03-26T21:01:44.323 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: [mds.foofs.smithi063.tzaznz{0:14474} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.63:6834/1380786213,v1:172.21.15.63:6835/1380786213] compat {c=[1],r=[1],i=[7ff]}] 2024-03-26T21:01:44.323 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: 2024-03-26T21:01:44.323 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: 2024-03-26T21:01:44.323 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Standby daemons: 2024-03-26T21:01:44.323 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: 2024-03-26T21:01:44.324 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: [mds.foofs.smithi174.gitdvk{-1:24307} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.174:6832/2611248035,v1:172.21.15.174:6833/2611248035] compat {c=[1],r=[1],i=[7ff]}] 2024-03-26T21:01:44.324 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:44 smithi174 ceph-mon[53014]: mon.smithi174@-1(???).osd e72 crush map has features 3314933000854323200, adjusting msgr requires 2024-03-26T21:01:44.324 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:44 smithi174 ceph-mon[53014]: mon.smithi174@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2024-03-26T21:01:44.324 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:44 smithi174 ceph-mon[53014]: mon.smithi174@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2024-03-26T21:01:44.324 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:44 smithi174 ceph-mon[53014]: mon.smithi174@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2024-03-26T21:01:44.324 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:44 smithi174 ceph-mon[53014]: mon.smithi174@-1(???).paxosservice(auth 1..30) refresh upgraded, format 0 -> 3 2024-03-26T21:01:44.324 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:44 smithi174 bash[52996]: 1a6d442692e789ee427d8d48fa89d6faa44467ae591f6c2107e29d458209b5ae 2024-03-26T21:01:44.324 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:44 smithi174 systemd[1]: Started Ceph mon.smithi174 for 0ed4dd4c-ebb2-11ee-95d0-87774f69a715. 2024-03-26T21:01:44.483 INFO:teuthology.orchestra.run.smithi063.stdout: 5 163617 11.52 MB/sec execute 218 sec latency 29.284 ms 2024-03-26T21:01:45.483 INFO:teuthology.orchestra.run.smithi063.stdout: 5 164049 11.54 MB/sec execute 219 sec latency 29.801 ms 2024-03-26T21:01:46.422 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:46 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mon metadata", "id": "smithi063"}]: dispatch 2024-03-26T21:01:46.422 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:46 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mon metadata", "id": "smithi174"}]: dispatch 2024-03-26T21:01:46.422 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:46 smithi063 ceph-mon[66934]: mon.smithi063 calling monitor election 2024-03-26T21:01:46.422 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:46 smithi063 ceph-mon[66934]: mon.smithi174 calling monitor election 2024-03-26T21:01:46.422 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:46 smithi063 ceph-mon[66934]: mon.smithi063 is new leader, mons smithi063,smithi174 in quorum (ranks 0,1) 2024-03-26T21:01:46.422 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:46 smithi063 ceph-mon[66934]: monmap epoch 3 2024-03-26T21:01:46.423 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:46 smithi063 ceph-mon[66934]: fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 2024-03-26T21:01:46.423 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:46 smithi063 ceph-mon[66934]: last_changed 2024-03-26T21:01:45.217515+0000 2024-03-26T21:01:46.423 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:46 smithi063 ceph-mon[66934]: created 2024-03-26T20:48:04.639095+0000 2024-03-26T21:01:46.423 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:46 smithi063 ceph-mon[66934]: min_mon_release 19 (squid) 2024-03-26T21:01:46.423 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:46 smithi063 ceph-mon[66934]: election_strategy: 1 2024-03-26T21:01:46.423 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:46 smithi063 ceph-mon[66934]: 0: [v2:172.21.15.63:3300/0,v1:172.21.15.63:6789/0] mon.smithi063 2024-03-26T21:01:46.423 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:46 smithi063 ceph-mon[66934]: 1: [v2:172.21.15.174:3300/0,v1:172.21.15.174:6789/0] mon.smithi174 2024-03-26T21:01:46.423 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:46 smithi063 ceph-mon[66934]: fsmap foofs:1 {0=foofs.smithi063.tzaznz=up:active} 1 up:standby 2024-03-26T21:01:46.423 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:46 smithi063 ceph-mon[66934]: osdmap e72: 8 total, 8 up, 8 in 2024-03-26T21:01:46.423 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:46 smithi063 ceph-mon[66934]: mgrmap e41: smithi063.lxhujt(active, since 19s), standbys: smithi174.sexoin 2024-03-26T21:01:46.423 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:46 smithi063 ceph-mon[66934]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-26T21:01:46.423 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:46 smithi063 ceph-mon[66934]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-26T21:01:46.423 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:46 smithi063 ceph-mon[66934]: daemon grafana.smithi063 on smithi063 is in error state 2024-03-26T21:01:46.423 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:46 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:01:46.424 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:46 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:01:46.424 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:46 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:01:46.424 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:46 smithi063 ceph-mon[66934]: pgmap v14: 97 pgs: 97 active+clean; 341 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.4 MiB/s wr, 529 op/s 2024-03-26T21:01:46.484 INFO:teuthology.orchestra.run.smithi063.stdout: 5 164593 11.53 MB/sec execute 220 sec latency 48.799 ms 2024-03-26T21:01:46.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:46 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mon metadata", "id": "smithi063"}]: dispatch 2024-03-26T21:01:46.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:46 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mon metadata", "id": "smithi174"}]: dispatch 2024-03-26T21:01:46.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:46 smithi174 ceph-mon[53014]: mon.smithi063 calling monitor election 2024-03-26T21:01:46.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:46 smithi174 ceph-mon[53014]: mon.smithi174 calling monitor election 2024-03-26T21:01:46.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:46 smithi174 ceph-mon[53014]: mon.smithi063 is new leader, mons smithi063,smithi174 in quorum (ranks 0,1) 2024-03-26T21:01:46.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:46 smithi174 ceph-mon[53014]: monmap epoch 3 2024-03-26T21:01:46.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:46 smithi174 ceph-mon[53014]: fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 2024-03-26T21:01:46.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:46 smithi174 ceph-mon[53014]: last_changed 2024-03-26T21:01:45.217515+0000 2024-03-26T21:01:46.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:46 smithi174 ceph-mon[53014]: created 2024-03-26T20:48:04.639095+0000 2024-03-26T21:01:46.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:46 smithi174 ceph-mon[53014]: min_mon_release 19 (squid) 2024-03-26T21:01:46.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:46 smithi174 ceph-mon[53014]: election_strategy: 1 2024-03-26T21:01:46.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:46 smithi174 ceph-mon[53014]: 0: [v2:172.21.15.63:3300/0,v1:172.21.15.63:6789/0] mon.smithi063 2024-03-26T21:01:46.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:46 smithi174 ceph-mon[53014]: 1: [v2:172.21.15.174:3300/0,v1:172.21.15.174:6789/0] mon.smithi174 2024-03-26T21:01:46.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:46 smithi174 ceph-mon[53014]: fsmap foofs:1 {0=foofs.smithi063.tzaznz=up:active} 1 up:standby 2024-03-26T21:01:46.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:46 smithi174 ceph-mon[53014]: osdmap e72: 8 total, 8 up, 8 in 2024-03-26T21:01:46.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:46 smithi174 ceph-mon[53014]: mgrmap e41: smithi063.lxhujt(active, since 19s), standbys: smithi174.sexoin 2024-03-26T21:01:46.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:46 smithi174 ceph-mon[53014]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-26T21:01:46.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:46 smithi174 ceph-mon[53014]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-26T21:01:46.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:46 smithi174 ceph-mon[53014]: daemon grafana.smithi063 on smithi063 is in error state 2024-03-26T21:01:46.550 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:46 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:01:46.550 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:46 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:01:46.550 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:46 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:01:46.550 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:46 smithi174 ceph-mon[53014]: pgmap v14: 97 pgs: 97 active+clean; 341 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.4 MiB/s wr, 529 op/s 2024-03-26T21:01:47.484 INFO:teuthology.orchestra.run.smithi063.stdout: 5 165129 11.53 MB/sec execute 221 sec latency 34.665 ms 2024-03-26T21:01:48.483 INFO:teuthology.orchestra.run.smithi063.stdout: 5 165659 11.56 MB/sec execute 222 sec latency 55.212 ms 2024-03-26T21:01:48.697 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:48 smithi063 ceph-mon[66934]: pgmap v15: 97 pgs: 97 active+clean; 340 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.5 MiB/s wr, 555 op/s 2024-03-26T21:01:49.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:48 smithi174 ceph-mon[53014]: pgmap v15: 97 pgs: 97 active+clean; 340 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.5 MiB/s wr, 555 op/s 2024-03-26T21:01:49.484 INFO:teuthology.orchestra.run.smithi063.stdout: 5 166245 11.59 MB/sec execute 223 sec latency 33.770 ms 2024-03-26T21:01:50.484 INFO:teuthology.orchestra.run.smithi063.stdout: 5 166567 11.57 MB/sec execute 224 sec latency 47.706 ms 2024-03-26T21:01:51.000 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:50 smithi063 ceph-mon[66934]: pgmap v16: 97 pgs: 97 active+clean; 340 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.6 MiB/s wr, 468 op/s 2024-03-26T21:01:51.000 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:50 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:01:51.001 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:50 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:01:51.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:50 smithi174 ceph-mon[53014]: pgmap v16: 97 pgs: 97 active+clean; 340 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.6 MiB/s wr, 468 op/s 2024-03-26T21:01:51.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:50 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:01:51.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:50 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:01:51.484 INFO:teuthology.orchestra.run.smithi063.stdout: 5 166859 11.56 MB/sec execute 225 sec latency 62.989 ms 2024-03-26T21:01:52.234 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:52 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:01:52.234 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:52 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:01:52.275 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:52 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:01:52.275 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:52 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:01:52.484 INFO:teuthology.orchestra.run.smithi063.stdout: 5 167130 11.53 MB/sec execute 226 sec latency 75.094 ms 2024-03-26T21:01:53.283 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:53 smithi063 ceph-mon[66934]: pgmap v17: 97 pgs: 97 active+clean; 349 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 7.2 MiB/s wr, 721 op/s 2024-03-26T21:01:53.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:53 smithi174 ceph-mon[53014]: pgmap v17: 97 pgs: 97 active+clean; 349 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 7.2 MiB/s wr, 721 op/s 2024-03-26T21:01:53.484 INFO:teuthology.orchestra.run.smithi063.stdout: 5 167449 11.54 MB/sec execute 227 sec latency 57.269 ms 2024-03-26T21:01:54.484 INFO:teuthology.orchestra.run.smithi063.stdout: 5 167760 11.52 MB/sec execute 228 sec latency 51.812 ms 2024-03-26T21:01:55.257 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:55 smithi063 ceph-mon[66934]: pgmap v18: 97 pgs: 97 active+clean; 350 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.6 MiB/s wr, 530 op/s 2024-03-26T21:01:55.257 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:55 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:01:55.257 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:55 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:01:55.257 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:55 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:01:55.257 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:55 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T21:01:55.257 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:55 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:01:55.257 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:55 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:01:55.257 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:55 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-26T21:01:55.257 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:55 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-26T21:01:55.257 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:55 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:01:55.484 INFO:teuthology.orchestra.run.smithi063.stdout: 5 168079 11.49 MB/sec execute 229 sec latency 94.918 ms 2024-03-26T21:01:55.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:55 smithi174 ceph-mon[53014]: pgmap v18: 97 pgs: 97 active+clean; 350 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.6 MiB/s wr, 530 op/s 2024-03-26T21:01:55.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:55 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:01:55.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:55 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:01:55.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:55 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:01:55.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:55 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T21:01:55.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:55 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:01:55.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:55 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:01:55.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:55 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-26T21:01:55.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:55 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-26T21:01:55.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:55 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:01:56.289 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:56 smithi063 ceph-mon[66934]: pgmap v19: 97 pgs: 97 active+clean; 350 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.8 MiB/s wr, 559 op/s 2024-03-26T21:01:56.289 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:56 smithi063 ceph-mon[66934]: pgmap v20: 97 pgs: 97 active+clean; 350 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.4 KiB/s rd, 7.0 MiB/s wr, 681 op/s 2024-03-26T21:01:56.289 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:56 smithi063 ceph-mon[66934]: Reconfiguring mon.smithi063 (monmap changed)... 2024-03-26T21:01:56.289 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:56 smithi063 ceph-mon[66934]: Reconfiguring daemon mon.smithi063 on smithi063 2024-03-26T21:01:56.289 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:56 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:01:56.484 INFO:teuthology.orchestra.run.smithi063.stdout: 5 168405 11.46 MB/sec execute 230 sec latency 39.979 ms 2024-03-26T21:01:56.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:56 smithi174 ceph-mon[53014]: pgmap v19: 97 pgs: 97 active+clean; 350 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.8 MiB/s wr, 559 op/s 2024-03-26T21:01:56.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:56 smithi174 ceph-mon[53014]: pgmap v20: 97 pgs: 97 active+clean; 350 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.4 KiB/s rd, 7.0 MiB/s wr, 681 op/s 2024-03-26T21:01:56.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:56 smithi174 ceph-mon[53014]: Reconfiguring mon.smithi063 (monmap changed)... 2024-03-26T21:01:56.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:56 smithi174 ceph-mon[53014]: Reconfiguring daemon mon.smithi063 on smithi063 2024-03-26T21:01:56.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:56 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:01:57.484 INFO:teuthology.orchestra.run.smithi063.stdout: 5 168678 11.44 MB/sec execute 231 sec latency 60.555 ms 2024-03-26T21:01:58.341 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:58 smithi063 ceph-mon[66934]: pgmap v21: 97 pgs: 97 active+clean; 342 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.3 KiB/s rd, 6.1 MiB/s wr, 599 op/s 2024-03-26T21:01:58.342 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:58 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:01:58.342 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:58 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:01:58.342 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:58 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi063.lxhujt", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-26T21:01:58.342 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:58 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-26T21:01:58.342 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:58 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:01:58.484 INFO:teuthology.orchestra.run.smithi063.stdout: 5 168980 11.45 MB/sec execute 232 sec latency 56.708 ms 2024-03-26T21:01:58.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:58 smithi174 ceph-mon[53014]: pgmap v21: 97 pgs: 97 active+clean; 342 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.3 KiB/s rd, 6.1 MiB/s wr, 599 op/s 2024-03-26T21:01:58.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:58 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:01:58.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:58 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:01:58.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:58 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi063.lxhujt", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-26T21:01:58.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:58 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-26T21:01:58.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:58 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:01:59.358 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:59 smithi063 ceph-mon[66934]: Reconfiguring mgr.smithi063.lxhujt (monmap changed)... 2024-03-26T21:01:59.359 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:01:59 smithi063 ceph-mon[66934]: Reconfiguring daemon mgr.smithi063.lxhujt on smithi063 2024-03-26T21:01:59.484 INFO:teuthology.orchestra.run.smithi063.stdout: 5 169254 11.44 MB/sec execute 233 sec latency 57.422 ms 2024-03-26T21:01:59.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:59 smithi174 ceph-mon[53014]: Reconfiguring mgr.smithi063.lxhujt (monmap changed)... 2024-03-26T21:01:59.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:01:59 smithi174 ceph-mon[53014]: Reconfiguring daemon mgr.smithi063.lxhujt on smithi063 2024-03-26T21:02:00.403 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:00 smithi063 ceph-mon[66934]: pgmap v22: 97 pgs: 97 active+clean; 342 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.3 KiB/s rd, 6.1 MiB/s wr, 599 op/s 2024-03-26T21:02:00.404 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:00 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:00.404 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:00 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:00.404 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:00 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi063", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-03-26T21:02:00.404 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:00 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:02:00.484 INFO:teuthology.orchestra.run.smithi063.stdout: 5 169558 11.43 MB/sec execute 234 sec latency 68.488 ms 2024-03-26T21:02:00.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:00 smithi174 ceph-mon[53014]: pgmap v22: 97 pgs: 97 active+clean; 342 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.3 KiB/s rd, 6.1 MiB/s wr, 599 op/s 2024-03-26T21:02:00.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:00 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:00.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:00 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:00.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:00 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi063", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-03-26T21:02:00.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:00 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:02:01.426 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:01 smithi063 ceph-mon[66934]: Reconfiguring crash.smithi063 (monmap changed)... 2024-03-26T21:02:01.426 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:01 smithi063 ceph-mon[66934]: Reconfiguring daemon crash.smithi063 on smithi063 2024-03-26T21:02:01.484 INFO:teuthology.orchestra.run.smithi063.stdout: 5 169872 11.41 MB/sec execute 235 sec latency 61.184 ms 2024-03-26T21:02:01.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:01 smithi174 ceph-mon[53014]: Reconfiguring crash.smithi063 (monmap changed)... 2024-03-26T21:02:01.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:01 smithi174 ceph-mon[53014]: Reconfiguring daemon crash.smithi063 on smithi063 2024-03-26T21:02:02.447 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:02 smithi063 ceph-mon[66934]: pgmap v23: 97 pgs: 97 active+clean; 342 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 4.3 KiB/s rd, 2.7 MiB/s wr, 269 op/s 2024-03-26T21:02:02.485 INFO:teuthology.orchestra.run.smithi063.stdout: 5 170106 11.39 MB/sec execute 236 sec latency 30.636 ms 2024-03-26T21:02:02.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:02 smithi174 ceph-mon[53014]: pgmap v23: 97 pgs: 97 active+clean; 342 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 4.3 KiB/s rd, 2.7 MiB/s wr, 269 op/s 2024-03-26T21:02:03.484 INFO:teuthology.orchestra.run.smithi063.stdout: 5 170416 11.38 MB/sec execute 237 sec latency 47.096 ms 2024-03-26T21:02:03.709 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:03 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:03.709 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:03 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:03.709 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:03 smithi063 ceph-mon[66934]: Reconfiguring osd.0 (monmap changed)... 2024-03-26T21:02:03.709 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:03 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-03-26T21:02:03.709 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:03 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:02:03.709 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:03 smithi063 ceph-mon[66934]: Reconfiguring daemon osd.0 on smithi063 2024-03-26T21:02:03.709 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:03 smithi063 ceph-mon[66934]: pgmap v24: 97 pgs: 97 active+clean; 344 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 4.8 MiB/s wr, 482 op/s 2024-03-26T21:02:04.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:03 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:04.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:03 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:04.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:03 smithi174 ceph-mon[53014]: Reconfiguring osd.0 (monmap changed)... 2024-03-26T21:02:04.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:03 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-03-26T21:02:04.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:03 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:02:04.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:03 smithi174 ceph-mon[53014]: Reconfiguring daemon osd.0 on smithi063 2024-03-26T21:02:04.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:03 smithi174 ceph-mon[53014]: pgmap v24: 97 pgs: 97 active+clean; 344 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 4.8 MiB/s wr, 482 op/s 2024-03-26T21:02:04.484 INFO:teuthology.orchestra.run.smithi063.stdout: 5 170712 11.35 MB/sec execute 238 sec latency 48.837 ms 2024-03-26T21:02:05.485 INFO:teuthology.orchestra.run.smithi063.stdout: 5 170941 11.34 MB/sec execute 239 sec latency 58.984 ms 2024-03-26T21:02:06.048 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:05 smithi063 ceph-mon[66934]: pgmap v25: 97 pgs: 97 active+clean; 344 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.4 MiB/s wr, 443 op/s 2024-03-26T21:02:06.048 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:05 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:06.048 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:05 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:06.048 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:05 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-03-26T21:02:06.048 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:05 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:02:06.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:05 smithi174 ceph-mon[53014]: pgmap v25: 97 pgs: 97 active+clean; 344 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.4 MiB/s wr, 443 op/s 2024-03-26T21:02:06.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:05 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:06.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:05 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:06.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:05 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-03-26T21:02:06.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:05 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:02:06.484 INFO:teuthology.orchestra.run.smithi063.stdout: 5 171193 11.34 MB/sec execute 240 sec latency 68.548 ms 2024-03-26T21:02:07.093 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:06 smithi063 ceph-mon[66934]: Reconfiguring osd.1 (monmap changed)... 2024-03-26T21:02:07.093 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:06 smithi063 ceph-mon[66934]: Reconfiguring daemon osd.1 on smithi063 2024-03-26T21:02:07.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:06 smithi174 ceph-mon[53014]: Reconfiguring osd.1 (monmap changed)... 2024-03-26T21:02:07.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:06 smithi174 ceph-mon[53014]: Reconfiguring daemon osd.1 on smithi063 2024-03-26T21:02:07.484 INFO:teuthology.orchestra.run.smithi063.stdout: 5 171437 11.31 MB/sec execute 241 sec latency 75.054 ms 2024-03-26T21:02:07.499 INFO:teuthology.orchestra.run.smithi063.stdout:true 2024-03-26T21:02:08.103 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:07 smithi063 ceph-mon[66934]: pgmap v26: 97 pgs: 97 active+clean; 339 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 5.5 MiB/s wr, 544 op/s 2024-03-26T21:02:08.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:07 smithi174 ceph-mon[53014]: pgmap v26: 97 pgs: 97 active+clean; 339 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 5.5 MiB/s wr, 544 op/s 2024-03-26T21:02:08.484 INFO:teuthology.orchestra.run.smithi063.stdout: 5 171780 11.27 MB/sec execute 242 sec latency 51.139 ms 2024-03-26T21:02:08.563 INFO:teuthology.orchestra.run.smithi063.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-26T21:02:08.563 INFO:teuthology.orchestra.run.smithi063.stdout:alertmanager.smithi063 smithi063 *:9093,9094 running (3m) 33s ago 12m 17.6M - 0.23.0 ba2b418f427c 6e160c94aac3 2024-03-26T21:02:08.563 INFO:teuthology.orchestra.run.smithi063.stdout:crash.smithi063 smithi063 running (12m) 33s ago 12m 6409k - 17.2.0 e1d6a67b021e 706c62f8af89 2024-03-26T21:02:08.563 INFO:teuthology.orchestra.run.smithi063.stdout:crash.smithi174 smithi174 running (11m) 18s ago 11m 6409k - 17.2.0 e1d6a67b021e 2c6bfe6d0ca6 2024-03-26T21:02:08.563 INFO:teuthology.orchestra.run.smithi063.stdout:grafana.smithi063 smithi063 *:3000 error 33s ago 12m - - 2024-03-26T21:02:08.563 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foofs.smithi063.tzaznz smithi063 running (6m) 33s ago 6m 37.2M - 17.2.0 e1d6a67b021e 10a989fc7954 2024-03-26T21:02:08.563 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foofs.smithi174.gitdvk smithi174 running (6m) 18s ago 6m 13.2M - 17.2.0 e1d6a67b021e 4339a5d98d1d 2024-03-26T21:02:08.563 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.smithi063.lxhujt smithi063 *:8443,9283,8765 running (2m) 33s ago 13m 477M - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 6d9d10330a90 2024-03-26T21:02:08.563 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.smithi174.sexoin smithi174 *:8443,9283,8765 running (84s) 18s ago 11m 449M - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 d7546233672d 2024-03-26T21:02:08.564 INFO:teuthology.orchestra.run.smithi063.stdout:mon.smithi063 smithi063 running (64s) 33s ago 13m 35.2M 2048M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 c644200344f8 2024-03-26T21:02:08.564 INFO:teuthology.orchestra.run.smithi063.stdout:mon.smithi174 smithi174 running (24s) 18s ago 11m 27.2M 2048M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 1a6d442692e7 2024-03-26T21:02:08.564 INFO:teuthology.orchestra.run.smithi063.stdout:nfs.foo.0.0.smithi063.fgmpwa smithi063 *:2049 running (6m) 33s ago 6m 275M - 4.0 e1d6a67b021e 3d6c0dd9ce51 2024-03-26T21:02:08.564 INFO:teuthology.orchestra.run.smithi063.stdout:nfs.foo.1.0.smithi174.pytptt smithi174 *:2049 running (6m) 18s ago 6m 86.3M - 4.0 e1d6a67b021e f220dbb449a3 2024-03-26T21:02:08.564 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.smithi063 smithi063 *:9100 running (2m) 33s ago 12m 12.6M - 1.3.1 1dbe0e931976 f52f41fd14c0 2024-03-26T21:02:08.564 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.smithi174 smithi174 *:9100 running (2m) 18s ago 11m 19.9M - 1.3.1 1dbe0e931976 0c027f39a4c8 2024-03-26T21:02:08.564 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0 smithi063 running (11m) 33s ago 11m 564M 1936M 17.2.0 e1d6a67b021e 761b1388d8fa 2024-03-26T21:02:08.564 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1 smithi063 running (10m) 33s ago 10m 481M 1936M 17.2.0 e1d6a67b021e 108275c17cad 2024-03-26T21:02:08.564 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2 smithi063 running (10m) 33s ago 10m 334M 1936M 17.2.0 e1d6a67b021e 107f685ad08d 2024-03-26T21:02:08.564 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3 smithi063 running (9m) 33s ago 9m 500M 1936M 17.2.0 e1d6a67b021e ab0f1a855957 2024-03-26T21:02:08.564 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4 smithi174 running (9m) 18s ago 9m 548M 2704M 17.2.0 e1d6a67b021e a0e877ecc06d 2024-03-26T21:02:08.564 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5 smithi174 running (8m) 18s ago 8m 349M 2704M 17.2.0 e1d6a67b021e 378c2aca7e71 2024-03-26T21:02:08.564 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6 smithi174 running (8m) 18s ago 8m 538M 2704M 17.2.0 e1d6a67b021e ffbb5b7ad3b6 2024-03-26T21:02:08.564 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7 smithi174 running (7m) 18s ago 7m 430M 2704M 17.2.0 e1d6a67b021e c1d57fcabb3f 2024-03-26T21:02:08.565 INFO:teuthology.orchestra.run.smithi063.stdout:prometheus.smithi063 smithi063 *:9095 running (90s) 33s ago 11m 62.0M - 2.33.4 514e6a882f6e 567eef202627 2024-03-26T21:02:09.107 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:08 smithi063 ceph-mon[66934]: from='client.34156 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:02:09.108 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:08 smithi063 ceph-mon[66934]: from='client.34160 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:02:09.108 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:08 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:09.108 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:08 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:09.108 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:08 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-03-26T21:02:09.108 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:08 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:02:09.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:08 smithi174 ceph-mon[53014]: from='client.34156 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:02:09.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:08 smithi174 ceph-mon[53014]: from='client.34160 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:02:09.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:08 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:09.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:08 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:09.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:08 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-03-26T21:02:09.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:08 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:02:09.308 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-03-26T21:02:09.309 INFO:teuthology.orchestra.run.smithi063.stdout: "mon": { 2024-03-26T21:02:09.309 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 2 2024-03-26T21:02:09.309 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:02:09.309 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr": { 2024-03-26T21:02:09.309 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 2 2024-03-26T21:02:09.309 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:02:09.309 INFO:teuthology.orchestra.run.smithi063.stdout: "osd": { 2024-03-26T21:02:09.309 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-03-26T21:02:09.309 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:02:09.309 INFO:teuthology.orchestra.run.smithi063.stdout: "mds": { 2024-03-26T21:02:09.309 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-26T21:02:09.310 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:02:09.310 INFO:teuthology.orchestra.run.smithi063.stdout: "overall": { 2024-03-26T21:02:09.310 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-03-26T21:02:09.310 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 4 2024-03-26T21:02:09.310 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-03-26T21:02:09.310 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-03-26T21:02:09.484 INFO:teuthology.orchestra.run.smithi063.stdout: 5 172031 11.25 MB/sec execute 243 sec latency 48.422 ms 2024-03-26T21:02:09.925 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-03-26T21:02:09.926 INFO:teuthology.orchestra.run.smithi063.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d", 2024-03-26T21:02:09.926 INFO:teuthology.orchestra.run.smithi063.stdout: "in_progress": true, 2024-03-26T21:02:09.926 INFO:teuthology.orchestra.run.smithi063.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-26T21:02:09.926 INFO:teuthology.orchestra.run.smithi063.stdout: "services_complete": [ 2024-03-26T21:02:09.926 INFO:teuthology.orchestra.run.smithi063.stdout: "mon", 2024-03-26T21:02:09.926 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr" 2024-03-26T21:02:09.926 INFO:teuthology.orchestra.run.smithi063.stdout: ], 2024-03-26T21:02:09.926 INFO:teuthology.orchestra.run.smithi063.stdout: "progress": "4/23 daemons upgraded", 2024-03-26T21:02:09.926 INFO:teuthology.orchestra.run.smithi063.stdout: "message": "Currently upgrading mon daemons", 2024-03-26T21:02:09.926 INFO:teuthology.orchestra.run.smithi063.stdout: "is_paused": false 2024-03-26T21:02:09.926 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-03-26T21:02:10.127 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:09 smithi063 ceph-mon[66934]: from='client.34164 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:02:10.128 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:09 smithi063 ceph-mon[66934]: Reconfiguring osd.2 (monmap changed)... 2024-03-26T21:02:10.128 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:09 smithi063 ceph-mon[66934]: Reconfiguring daemon osd.2 on smithi063 2024-03-26T21:02:10.128 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:09 smithi063 ceph-mon[66934]: pgmap v27: 97 pgs: 97 active+clean; 339 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.7 MiB/s wr, 350 op/s 2024-03-26T21:02:10.128 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:09 smithi063 ceph-mon[66934]: from='client.? 172.21.15.63:0/328486467' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:02:10.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:09 smithi174 ceph-mon[53014]: from='client.34164 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:02:10.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:09 smithi174 ceph-mon[53014]: Reconfiguring osd.2 (monmap changed)... 2024-03-26T21:02:10.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:09 smithi174 ceph-mon[53014]: Reconfiguring daemon osd.2 on smithi063 2024-03-26T21:02:10.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:09 smithi174 ceph-mon[53014]: pgmap v27: 97 pgs: 97 active+clean; 339 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.7 MiB/s wr, 350 op/s 2024-03-26T21:02:10.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:09 smithi174 ceph-mon[53014]: from='client.? 172.21.15.63:0/328486467' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:02:10.484 INFO:teuthology.orchestra.run.smithi063.stdout: 5 172362 11.25 MB/sec execute 244 sec latency 43.741 ms 2024-03-26T21:02:10.603 INFO:teuthology.orchestra.run.smithi063.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-26T21:02:10.603 INFO:teuthology.orchestra.run.smithi063.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-26T21:02:10.603 INFO:teuthology.orchestra.run.smithi063.stdout: daemon grafana.smithi063 on smithi063 is in error state 2024-03-26T21:02:11.136 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:10 smithi063 ceph-mon[66934]: from='client.44115 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:02:11.137 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:10 smithi063 ceph-mon[66934]: from='client.? 172.21.15.63:0/3399592218' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-26T21:02:11.137 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:10 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:02:11.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:10 smithi174 ceph-mon[53014]: from='client.44115 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:02:11.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:10 smithi174 ceph-mon[53014]: from='client.? 172.21.15.63:0/3399592218' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-26T21:02:11.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:10 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:02:11.484 INFO:teuthology.orchestra.run.smithi063.stdout: 5 172817 11.27 MB/sec execute 245 sec latency 38.071 ms 2024-03-26T21:02:12.155 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:11 smithi063 ceph-mon[66934]: pgmap v28: 97 pgs: 97 active+clean; 339 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.7 MiB/s wr, 350 op/s 2024-03-26T21:02:12.155 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:11 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:12.155 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:11 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:12.155 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:11 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-03-26T21:02:12.155 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:11 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:02:12.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:11 smithi174 ceph-mon[53014]: pgmap v28: 97 pgs: 97 active+clean; 339 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.7 MiB/s wr, 350 op/s 2024-03-26T21:02:12.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:11 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:12.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:11 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:12.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:11 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-03-26T21:02:12.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:11 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:02:12.484 INFO:teuthology.orchestra.run.smithi063.stdout: 5 173210 11.28 MB/sec execute 246 sec latency 66.149 ms 2024-03-26T21:02:13.178 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:12 smithi063 ceph-mon[66934]: Reconfiguring osd.3 (monmap changed)... 2024-03-26T21:02:13.178 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:12 smithi063 ceph-mon[66934]: Reconfiguring daemon osd.3 on smithi063 2024-03-26T21:02:13.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:12 smithi174 ceph-mon[53014]: Reconfiguring osd.3 (monmap changed)... 2024-03-26T21:02:13.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:12 smithi174 ceph-mon[53014]: Reconfiguring daemon osd.3 on smithi063 2024-03-26T21:02:13.484 INFO:teuthology.orchestra.run.smithi063.stdout: 5 173507 11.26 MB/sec execute 247 sec latency 58.273 ms 2024-03-26T21:02:14.193 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:13 smithi063 ceph-mon[66934]: pgmap v29: 97 pgs: 97 active+clean; 343 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.6 MiB/s wr, 545 op/s 2024-03-26T21:02:14.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:13 smithi174 ceph-mon[53014]: pgmap v29: 97 pgs: 97 active+clean; 343 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.6 MiB/s wr, 545 op/s 2024-03-26T21:02:14.485 INFO:teuthology.orchestra.run.smithi063.stdout: 5 173723 11.22 MB/sec execute 248 sec latency 44.825 ms 2024-03-26T21:02:15.466 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:15 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:15.466 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:15 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:15.466 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:15 smithi063 ceph-mon[66934]: Reconfiguring mds.foofs.smithi063.tzaznz (monmap changed)... 2024-03-26T21:02:15.466 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:15 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi063.tzaznz", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-26T21:02:15.466 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:15 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:02:15.466 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:15 smithi063 ceph-mon[66934]: Reconfiguring daemon mds.foofs.smithi063.tzaznz on smithi063 2024-03-26T21:02:15.466 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:15 smithi063 ceph-mon[66934]: pgmap v30: 97 pgs: 97 active+clean; 343 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.8 MiB/s wr, 368 op/s 2024-03-26T21:02:15.484 INFO:teuthology.orchestra.run.smithi063.stdout: 5 174011 11.22 MB/sec execute 249 sec latency 54.857 ms 2024-03-26T21:02:15.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:15 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:15.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:15 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:15.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:15 smithi174 ceph-mon[53014]: Reconfiguring mds.foofs.smithi063.tzaznz (monmap changed)... 2024-03-26T21:02:15.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:15 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi063.tzaznz", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-26T21:02:15.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:15 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:02:15.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:15 smithi174 ceph-mon[53014]: Reconfiguring daemon mds.foofs.smithi063.tzaznz on smithi063 2024-03-26T21:02:15.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:15 smithi174 ceph-mon[53014]: pgmap v30: 97 pgs: 97 active+clean; 343 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.8 MiB/s wr, 368 op/s 2024-03-26T21:02:16.485 INFO:teuthology.orchestra.run.smithi063.stdout: 5 174312 11.20 MB/sec execute 250 sec latency 53.348 ms 2024-03-26T21:02:17.485 INFO:teuthology.orchestra.run.smithi063.stdout: 5 174631 11.20 MB/sec execute 251 sec latency 41.539 ms 2024-03-26T21:02:18.296 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:18 smithi063 ceph-mon[66934]: pgmap v31: 97 pgs: 97 active+clean; 348 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.8 MiB/s wr, 554 op/s 2024-03-26T21:02:18.297 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:18 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:18.297 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:18 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:18.297 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:18 smithi063 ceph-mon[66934]: Reconfiguring crash.smithi174 (monmap changed)... 2024-03-26T21:02:18.297 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:18 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi174", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-03-26T21:02:18.297 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:18 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:02:18.297 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:18 smithi063 ceph-mon[66934]: Reconfiguring daemon crash.smithi174 on smithi174 2024-03-26T21:02:18.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:18 smithi174 ceph-mon[53014]: pgmap v31: 97 pgs: 97 active+clean; 348 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.8 MiB/s wr, 554 op/s 2024-03-26T21:02:18.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:18 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:18.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:18 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:18.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:18 smithi174 ceph-mon[53014]: Reconfiguring crash.smithi174 (monmap changed)... 2024-03-26T21:02:18.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:18 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi174", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-03-26T21:02:18.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:18 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:02:18.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:18 smithi174 ceph-mon[53014]: Reconfiguring daemon crash.smithi174 on smithi174 2024-03-26T21:02:18.485 INFO:teuthology.orchestra.run.smithi063.stdout: 5 174939 11.19 MB/sec execute 252 sec latency 60.723 ms 2024-03-26T21:02:19.485 INFO:teuthology.orchestra.run.smithi063.stdout: 5 175282 11.16 MB/sec execute 253 sec latency 55.590 ms 2024-03-26T21:02:19.834 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:19 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:19.834 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:19 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:19.834 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:19 smithi174 ceph-mon[53014]: Reconfiguring mgr.smithi174.sexoin (monmap changed)... 2024-03-26T21:02:19.834 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:19 smithi174 ceph-mon[53014]: pgmap v32: 97 pgs: 97 active+clean; 348 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 3.9 MiB/s wr, 380 op/s 2024-03-26T21:02:19.834 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:19 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi174.sexoin", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-26T21:02:19.834 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:19 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-26T21:02:19.834 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:19 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:02:19.835 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:19 smithi174 ceph-mon[53014]: Reconfiguring daemon mgr.smithi174.sexoin on smithi174 2024-03-26T21:02:20.069 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:19 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:20.069 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:19 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:20.069 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:19 smithi063 ceph-mon[66934]: Reconfiguring mgr.smithi174.sexoin (monmap changed)... 2024-03-26T21:02:20.069 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:19 smithi063 ceph-mon[66934]: pgmap v32: 97 pgs: 97 active+clean; 348 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 3.9 MiB/s wr, 380 op/s 2024-03-26T21:02:20.070 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:19 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi174.sexoin", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-26T21:02:20.070 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:19 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-26T21:02:20.070 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:19 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:02:20.070 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:19 smithi063 ceph-mon[66934]: Reconfiguring daemon mgr.smithi174.sexoin on smithi174 2024-03-26T21:02:20.485 INFO:teuthology.orchestra.run.smithi063.stdout: 5 175725 11.15 MB/sec execute 254 sec latency 71.947 ms 2024-03-26T21:02:21.485 INFO:teuthology.orchestra.run.smithi063.stdout: 5 176252 11.19 MB/sec execute 255 sec latency 61.587 ms 2024-03-26T21:02:21.836 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:21 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:21.836 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:21 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:21.836 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:21 smithi063 ceph-mon[66934]: Reconfiguring mon.smithi174 (monmap changed)... 2024-03-26T21:02:21.836 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:21 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-26T21:02:21.836 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:21 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-26T21:02:21.836 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:21 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:02:21.836 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:21 smithi063 ceph-mon[66934]: Reconfiguring daemon mon.smithi174 on smithi174 2024-03-26T21:02:21.836 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:21 smithi063 ceph-mon[66934]: pgmap v33: 97 pgs: 97 active+clean; 348 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 3.9 MiB/s wr, 380 op/s 2024-03-26T21:02:22.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:21 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:22.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:21 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:22.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:21 smithi174 ceph-mon[53014]: Reconfiguring mon.smithi174 (monmap changed)... 2024-03-26T21:02:22.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:21 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-26T21:02:22.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:21 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-26T21:02:22.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:21 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:02:22.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:21 smithi174 ceph-mon[53014]: Reconfiguring daemon mon.smithi174 on smithi174 2024-03-26T21:02:22.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:21 smithi174 ceph-mon[53014]: pgmap v33: 97 pgs: 97 active+clean; 348 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 3.9 MiB/s wr, 380 op/s 2024-03-26T21:02:22.485 INFO:teuthology.orchestra.run.smithi063.stdout: 5 176719 11.20 MB/sec execute 256 sec latency 78.805 ms 2024-03-26T21:02:23.485 INFO:teuthology.orchestra.run.smithi063.stdout: 5 177111 11.19 MB/sec execute 257 sec latency 40.898 ms 2024-03-26T21:02:23.857 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:23 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:23.857 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:23 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:23.857 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:23 smithi063 ceph-mon[66934]: Reconfiguring osd.4 (monmap changed)... 2024-03-26T21:02:23.857 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:23 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-03-26T21:02:23.857 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:23 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:02:23.857 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:23 smithi063 ceph-mon[66934]: Reconfiguring daemon osd.4 on smithi174 2024-03-26T21:02:23.857 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:23 smithi063 ceph-mon[66934]: pgmap v34: 97 pgs: 97 active+clean; 353 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 6.2 MiB/s wr, 612 op/s 2024-03-26T21:02:24.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:23 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:24.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:23 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:24.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:23 smithi174 ceph-mon[53014]: Reconfiguring osd.4 (monmap changed)... 2024-03-26T21:02:24.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:23 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-03-26T21:02:24.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:23 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:02:24.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:23 smithi174 ceph-mon[53014]: Reconfiguring daemon osd.4 on smithi174 2024-03-26T21:02:24.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:23 smithi174 ceph-mon[53014]: pgmap v34: 97 pgs: 97 active+clean; 353 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 6.2 MiB/s wr, 612 op/s 2024-03-26T21:02:24.485 INFO:teuthology.orchestra.run.smithi063.stdout: 5 177505 11.19 MB/sec execute 258 sec latency 33.072 ms 2024-03-26T21:02:25.485 INFO:teuthology.orchestra.run.smithi063.stdout: 5 177914 11.18 MB/sec execute 259 sec latency 32.372 ms 2024-03-26T21:02:25.650 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:25 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:25.650 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:25 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:25.650 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:25 smithi063 ceph-mon[66934]: Reconfiguring osd.5 (monmap changed)... 2024-03-26T21:02:25.900 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:25 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-03-26T21:02:25.900 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:25 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:02:25.900 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:25 smithi063 ceph-mon[66934]: Reconfiguring daemon osd.5 on smithi174 2024-03-26T21:02:25.900 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:25 smithi063 ceph-mon[66934]: pgmap v35: 97 pgs: 97 active+clean; 353 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.3 MiB/s wr, 418 op/s 2024-03-26T21:02:25.900 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:25 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:02:26.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:25 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:26.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:25 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:26.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:25 smithi174 ceph-mon[53014]: Reconfiguring osd.5 (monmap changed)... 2024-03-26T21:02:26.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:25 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-03-26T21:02:26.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:25 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:02:26.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:25 smithi174 ceph-mon[53014]: Reconfiguring daemon osd.5 on smithi174 2024-03-26T21:02:26.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:25 smithi174 ceph-mon[53014]: pgmap v35: 97 pgs: 97 active+clean; 353 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.3 MiB/s wr, 418 op/s 2024-03-26T21:02:26.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:25 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:02:26.486 INFO:teuthology.orchestra.run.smithi063.stdout: 5 178365 11.20 MB/sec execute 260 sec latency 38.337 ms 2024-03-26T21:02:27.485 INFO:teuthology.orchestra.run.smithi063.stdout: 5 178824 11.19 MB/sec execute 261 sec latency 35.769 ms 2024-03-26T21:02:27.672 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:27 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:27.672 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:27 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:27.672 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:27 smithi063 ceph-mon[66934]: Reconfiguring osd.6 (monmap changed)... 2024-03-26T21:02:27.672 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:27 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-03-26T21:02:27.672 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:27 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:02:27.672 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:27 smithi063 ceph-mon[66934]: Reconfiguring daemon osd.6 on smithi174 2024-03-26T21:02:27.672 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:27 smithi063 ceph-mon[66934]: pgmap v36: 97 pgs: 97 active+clean; 348 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 9.5 KiB/s rd, 6.9 MiB/s wr, 686 op/s 2024-03-26T21:02:27.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:27 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:27.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:27 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:27.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:27 smithi174 ceph-mon[53014]: Reconfiguring osd.6 (monmap changed)... 2024-03-26T21:02:27.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:27 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-03-26T21:02:27.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:27 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:02:27.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:27 smithi174 ceph-mon[53014]: Reconfiguring daemon osd.6 on smithi174 2024-03-26T21:02:27.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:27 smithi174 ceph-mon[53014]: pgmap v36: 97 pgs: 97 active+clean; 348 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 9.5 KiB/s rd, 6.9 MiB/s wr, 686 op/s 2024-03-26T21:02:28.485 INFO:teuthology.orchestra.run.smithi063.stdout: 5 179283 11.18 MB/sec execute 262 sec latency 32.436 ms 2024-03-26T21:02:29.485 INFO:teuthology.orchestra.run.smithi063.stdout: 5 179746 11.20 MB/sec execute 263 sec latency 47.373 ms 2024-03-26T21:02:29.902 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:29 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:29.902 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:29 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:29.902 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:29 smithi063 ceph-mon[66934]: Reconfiguring osd.7 (monmap changed)... 2024-03-26T21:02:29.902 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:29 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-03-26T21:02:29.902 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:29 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:02:29.902 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:29 smithi063 ceph-mon[66934]: Reconfiguring daemon osd.7 on smithi174 2024-03-26T21:02:29.902 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:29 smithi063 ceph-mon[66934]: pgmap v37: 97 pgs: 97 active+clean; 348 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.9 MiB/s wr, 500 op/s 2024-03-26T21:02:30.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:29 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:30.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:29 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:30.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:29 smithi174 ceph-mon[53014]: Reconfiguring osd.7 (monmap changed)... 2024-03-26T21:02:30.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:29 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-03-26T21:02:30.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:29 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:02:30.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:29 smithi174 ceph-mon[53014]: Reconfiguring daemon osd.7 on smithi174 2024-03-26T21:02:30.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:29 smithi174 ceph-mon[53014]: pgmap v37: 97 pgs: 97 active+clean; 348 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.9 MiB/s wr, 500 op/s 2024-03-26T21:02:30.486 INFO:teuthology.orchestra.run.smithi063.stdout: 5 180210 11.22 MB/sec execute 264 sec latency 42.056 ms 2024-03-26T21:02:31.485 INFO:teuthology.orchestra.run.smithi063.stdout: 5 180600 11.21 MB/sec execute 265 sec latency 47.202 ms 2024-03-26T21:02:31.691 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:31 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:31.691 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:31 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:31.691 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:31 smithi063 ceph-mon[66934]: Reconfiguring mds.foofs.smithi174.gitdvk (monmap changed)... 2024-03-26T21:02:31.691 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:31 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi174.gitdvk", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-26T21:02:31.691 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:31 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:02:31.691 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:31 smithi063 ceph-mon[66934]: Reconfiguring daemon mds.foofs.smithi174.gitdvk on smithi174 2024-03-26T21:02:31.691 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:31 smithi063 ceph-mon[66934]: pgmap v38: 97 pgs: 97 active+clean; 348 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.9 MiB/s wr, 500 op/s 2024-03-26T21:02:32.024 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:31 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:32.024 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:31 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:32.025 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:31 smithi174 ceph-mon[53014]: Reconfiguring mds.foofs.smithi174.gitdvk (monmap changed)... 2024-03-26T21:02:32.025 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:31 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi174.gitdvk", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-26T21:02:32.025 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:31 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:02:32.025 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:31 smithi174 ceph-mon[53014]: Reconfiguring daemon mds.foofs.smithi174.gitdvk on smithi174 2024-03-26T21:02:32.025 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:31 smithi174 ceph-mon[53014]: pgmap v38: 97 pgs: 97 active+clean; 348 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.9 MiB/s wr, 500 op/s 2024-03-26T21:02:32.486 INFO:teuthology.orchestra.run.smithi063.stdout: 5 180861 11.18 MB/sec execute 266 sec latency 54.904 ms 2024-03-26T21:02:33.478 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:33 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:33.478 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:33 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:33.478 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:33 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:02:33.478 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:33 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:02:33.478 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:33 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:02:33.478 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:33 smithi063 ceph-mon[66934]: Upgrade: Setting container_image for all mon 2024-03-26T21:02:33.478 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:33 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:33.478 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:33 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi063"}]: dispatch 2024-03-26T21:02:33.478 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:33 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi063"}]': finished 2024-03-26T21:02:33.478 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:33 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi174"}]: dispatch 2024-03-26T21:02:33.479 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:33 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi174"}]': finished 2024-03-26T21:02:33.479 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:33 smithi063 ceph-mon[66934]: pgmap v39: 97 pgs: 97 active+clean; 347 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 10 KiB/s rd, 7.6 MiB/s wr, 761 op/s 2024-03-26T21:02:33.485 INFO:teuthology.orchestra.run.smithi063.stdout: 5 181205 11.18 MB/sec execute 267 sec latency 44.160 ms 2024-03-26T21:02:33.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:33 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:33.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:33 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:33.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:33 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:02:33.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:33 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:02:33.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:33 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:02:33.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:33 smithi174 ceph-mon[53014]: Upgrade: Setting container_image for all mon 2024-03-26T21:02:33.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:33 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:33.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:33 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi063"}]: dispatch 2024-03-26T21:02:33.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:33 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi063"}]': finished 2024-03-26T21:02:33.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:33 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi174"}]: dispatch 2024-03-26T21:02:33.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:33 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi174"}]': finished 2024-03-26T21:02:33.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:33 smithi174 ceph-mon[53014]: pgmap v39: 97 pgs: 97 active+clean; 347 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 10 KiB/s rd, 7.6 MiB/s wr, 761 op/s 2024-03-26T21:02:34.486 INFO:teuthology.orchestra.run.smithi063.stdout: 5 181493 11.17 MB/sec execute 268 sec latency 69.522 ms 2024-03-26T21:02:35.486 INFO:teuthology.orchestra.run.smithi063.stdout: 5 181892 11.18 MB/sec execute 269 sec latency 39.587 ms 2024-03-26T21:02:36.276 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:36 smithi063 ceph-mon[66934]: pgmap v40: 97 pgs: 97 active+clean; 347 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.3 MiB/s wr, 529 op/s 2024-03-26T21:02:36.276 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:36 smithi063 ceph-mon[66934]: Upgrade: Updating crash.smithi063 (1/2) 2024-03-26T21:02:36.276 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:36 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:36.276 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:36 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi063", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-03-26T21:02:36.276 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:36 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:02:36.276 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:36 smithi063 ceph-mon[66934]: Deploying daemon crash.smithi063 on smithi063 2024-03-26T21:02:36.486 INFO:teuthology.orchestra.run.smithi063.stdout: 5 182585 11.18 MB/sec execute 270 sec latency 29.871 ms 2024-03-26T21:02:36.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:36 smithi174 ceph-mon[53014]: pgmap v40: 97 pgs: 97 active+clean; 347 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.3 MiB/s wr, 529 op/s 2024-03-26T21:02:36.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:36 smithi174 ceph-mon[53014]: Upgrade: Updating crash.smithi063 (1/2) 2024-03-26T21:02:36.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:36 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:36.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:36 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi063", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-03-26T21:02:36.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:36 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:02:36.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:36 smithi174 ceph-mon[53014]: Deploying daemon crash.smithi063 on smithi063 2024-03-26T21:02:37.486 INFO:teuthology.orchestra.run.smithi063.stdout: 5 183308 11.22 MB/sec execute 271 sec latency 25.873 ms 2024-03-26T21:02:38.313 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:38 smithi063 ceph-mon[66934]: pgmap v41: 97 pgs: 97 active+clean; 351 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 7.7 MiB/s wr, 782 op/s 2024-03-26T21:02:38.486 INFO:teuthology.orchestra.run.smithi063.stdout: 5 183630 11.23 MB/sec execute 272 sec latency 77.892 ms 2024-03-26T21:02:38.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:38 smithi174 ceph-mon[53014]: pgmap v41: 97 pgs: 97 active+clean; 351 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 7.7 MiB/s wr, 782 op/s 2024-03-26T21:02:39.491 INFO:teuthology.orchestra.run.smithi063.stdout: 5 183937 11.21 MB/sec execute 273 sec latency 84.136 ms 2024-03-26T21:02:40.345 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:40 smithi063 ceph-mon[66934]: pgmap v42: 97 pgs: 97 active+clean; 351 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.0 MiB/s wr, 513 op/s 2024-03-26T21:02:40.496 INFO:teuthology.orchestra.run.smithi063.stdout: 5 184205 11.19 MB/sec execute 274 sec latency 40.260 ms 2024-03-26T21:02:40.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:40 smithi174 ceph-mon[53014]: pgmap v42: 97 pgs: 97 active+clean; 351 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.0 MiB/s wr, 513 op/s 2024-03-26T21:02:41.257 INFO:teuthology.orchestra.run.smithi063.stdout:true 2024-03-26T21:02:41.380 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:41 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:02:41.496 INFO:teuthology.orchestra.run.smithi063.stdout: 5 184547 11.19 MB/sec execute 275 sec latency 56.395 ms 2024-03-26T21:02:41.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:41 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:02:42.384 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:42 smithi063 ceph-mon[66934]: pgmap v43: 97 pgs: 97 active+clean; 351 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.1 MiB/s wr, 513 op/s 2024-03-26T21:02:42.385 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:42 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:42.385 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:42 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:42.442 INFO:teuthology.orchestra.run.smithi063.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-26T21:02:42.442 INFO:teuthology.orchestra.run.smithi063.stdout:alertmanager.smithi063 smithi063 *:9093,9094 running (3m) 67s ago 13m 17.6M - 0.23.0 ba2b418f427c 6e160c94aac3 2024-03-26T21:02:42.443 INFO:teuthology.orchestra.run.smithi063.stdout:crash.smithi063 smithi063 starting - - - - 2024-03-26T21:02:42.443 INFO:teuthology.orchestra.run.smithi063.stdout:crash.smithi174 smithi174 running (12m) 52s ago 12m 6409k - 17.2.0 e1d6a67b021e 2c6bfe6d0ca6 2024-03-26T21:02:42.443 INFO:teuthology.orchestra.run.smithi063.stdout:grafana.smithi063 smithi063 *:3000 error 67s ago 12m - - 2024-03-26T21:02:42.443 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foofs.smithi063.tzaznz smithi063 running (7m) 67s ago 7m 37.2M - 17.2.0 e1d6a67b021e 10a989fc7954 2024-03-26T21:02:42.443 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foofs.smithi174.gitdvk smithi174 running (7m) 52s ago 7m 13.2M - 17.2.0 e1d6a67b021e 4339a5d98d1d 2024-03-26T21:02:42.443 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.smithi063.lxhujt smithi063 *:8443,9283,8765 running (2m) 67s ago 14m 477M - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 6d9d10330a90 2024-03-26T21:02:42.443 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.smithi174.sexoin smithi174 *:8443,9283,8765 running (118s) 52s ago 12m 449M - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 d7546233672d 2024-03-26T21:02:42.443 INFO:teuthology.orchestra.run.smithi063.stdout:mon.smithi063 smithi063 running (98s) 67s ago 14m 35.2M 2048M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 c644200344f8 2024-03-26T21:02:42.443 INFO:teuthology.orchestra.run.smithi063.stdout:mon.smithi174 smithi174 running (58s) 52s ago 12m 27.2M 2048M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 1a6d442692e7 2024-03-26T21:02:42.443 INFO:teuthology.orchestra.run.smithi063.stdout:nfs.foo.0.0.smithi063.fgmpwa smithi063 *:2049 running (6m) 67s ago 6m 275M - 4.0 e1d6a67b021e 3d6c0dd9ce51 2024-03-26T21:02:42.443 INFO:teuthology.orchestra.run.smithi063.stdout:nfs.foo.1.0.smithi174.pytptt smithi174 *:2049 running (6m) 52s ago 6m 86.3M - 4.0 e1d6a67b021e f220dbb449a3 2024-03-26T21:02:42.443 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.smithi063 smithi063 *:9100 running (3m) 67s ago 12m 12.6M - 1.3.1 1dbe0e931976 f52f41fd14c0 2024-03-26T21:02:42.443 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.smithi174 smithi174 *:9100 running (3m) 52s ago 12m 19.9M - 1.3.1 1dbe0e931976 0c027f39a4c8 2024-03-26T21:02:42.443 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0 smithi063 running (11m) 67s ago 11m 564M 1936M 17.2.0 e1d6a67b021e 761b1388d8fa 2024-03-26T21:02:42.443 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1 smithi063 running (11m) 67s ago 11m 481M 1936M 17.2.0 e1d6a67b021e 108275c17cad 2024-03-26T21:02:42.444 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2 smithi063 running (10m) 67s ago 10m 334M 1936M 17.2.0 e1d6a67b021e 107f685ad08d 2024-03-26T21:02:42.444 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3 smithi063 running (10m) 67s ago 10m 500M 1936M 17.2.0 e1d6a67b021e ab0f1a855957 2024-03-26T21:02:42.444 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4 smithi174 running (9m) 52s ago 9m 548M 2704M 17.2.0 e1d6a67b021e a0e877ecc06d 2024-03-26T21:02:42.444 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5 smithi174 running (9m) 52s ago 9m 349M 2704M 17.2.0 e1d6a67b021e 378c2aca7e71 2024-03-26T21:02:42.444 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6 smithi174 running (8m) 52s ago 8m 538M 2704M 17.2.0 e1d6a67b021e ffbb5b7ad3b6 2024-03-26T21:02:42.444 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7 smithi174 running (8m) 52s ago 8m 430M 2704M 17.2.0 e1d6a67b021e c1d57fcabb3f 2024-03-26T21:02:42.444 INFO:teuthology.orchestra.run.smithi063.stdout:prometheus.smithi063 smithi063 *:9095 running (2m) 67s ago 11m 62.0M - 2.33.4 514e6a882f6e 567eef202627 2024-03-26T21:02:42.496 INFO:teuthology.orchestra.run.smithi063.stdout: 5 185140 11.22 MB/sec execute 276 sec latency 28.009 ms 2024-03-26T21:02:42.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:42 smithi174 ceph-mon[53014]: pgmap v43: 97 pgs: 97 active+clean; 351 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.1 MiB/s wr, 513 op/s 2024-03-26T21:02:42.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:42 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:42.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:42 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:43.148 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-03-26T21:02:43.148 INFO:teuthology.orchestra.run.smithi063.stdout: "mon": { 2024-03-26T21:02:43.148 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 2 2024-03-26T21:02:43.148 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:02:43.149 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr": { 2024-03-26T21:02:43.149 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 2 2024-03-26T21:02:43.149 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:02:43.149 INFO:teuthology.orchestra.run.smithi063.stdout: "osd": { 2024-03-26T21:02:43.149 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-03-26T21:02:43.149 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:02:43.149 INFO:teuthology.orchestra.run.smithi063.stdout: "mds": { 2024-03-26T21:02:43.149 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-26T21:02:43.149 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:02:43.149 INFO:teuthology.orchestra.run.smithi063.stdout: "overall": { 2024-03-26T21:02:43.149 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-03-26T21:02:43.149 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 4 2024-03-26T21:02:43.149 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-03-26T21:02:43.149 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-03-26T21:02:43.392 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:43 smithi063 ceph-mon[66934]: from='client.34178 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:02:43.392 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:43 smithi063 ceph-mon[66934]: from='client.34182 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:02:43.392 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:43 smithi063 ceph-mon[66934]: from='client.? 172.21.15.63:0/3814697464' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:02:43.496 INFO:teuthology.orchestra.run.smithi063.stdout: 5 185547 11.21 MB/sec execute 277 sec latency 43.874 ms 2024-03-26T21:02:43.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:43 smithi174 ceph-mon[53014]: from='client.34178 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:02:43.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:43 smithi174 ceph-mon[53014]: from='client.34182 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:02:43.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:43 smithi174 ceph-mon[53014]: from='client.? 172.21.15.63:0/3814697464' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:02:43.706 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-03-26T21:02:43.707 INFO:teuthology.orchestra.run.smithi063.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d", 2024-03-26T21:02:43.707 INFO:teuthology.orchestra.run.smithi063.stdout: "in_progress": true, 2024-03-26T21:02:43.707 INFO:teuthology.orchestra.run.smithi063.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-26T21:02:43.707 INFO:teuthology.orchestra.run.smithi063.stdout: "services_complete": [ 2024-03-26T21:02:43.707 INFO:teuthology.orchestra.run.smithi063.stdout: "mon", 2024-03-26T21:02:43.707 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr" 2024-03-26T21:02:43.707 INFO:teuthology.orchestra.run.smithi063.stdout: ], 2024-03-26T21:02:43.707 INFO:teuthology.orchestra.run.smithi063.stdout: "progress": "4/23 daemons upgraded", 2024-03-26T21:02:43.707 INFO:teuthology.orchestra.run.smithi063.stdout: "message": "Currently upgrading crash daemons", 2024-03-26T21:02:43.707 INFO:teuthology.orchestra.run.smithi063.stdout: "is_paused": false 2024-03-26T21:02:43.707 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-03-26T21:02:44.397 INFO:teuthology.orchestra.run.smithi063.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-26T21:02:44.398 INFO:teuthology.orchestra.run.smithi063.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-26T21:02:44.398 INFO:teuthology.orchestra.run.smithi063.stdout: daemon grafana.smithi063 on smithi063 is in error state 2024-03-26T21:02:44.415 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:44 smithi063 ceph-mon[66934]: from='client.44125 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:02:44.415 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:44 smithi063 ceph-mon[66934]: pgmap v44: 97 pgs: 97 active+clean; 354 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 7.6 MiB/s wr, 746 op/s 2024-03-26T21:02:44.415 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:44 smithi063 ceph-mon[66934]: from='client.34194 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:02:44.415 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:44 smithi063 ceph-mon[66934]: Upgrade: Updating crash.smithi174 (2/2) 2024-03-26T21:02:44.415 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:44 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:44.415 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:44 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi174", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-03-26T21:02:44.415 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:44 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:02:44.415 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:44 smithi063 ceph-mon[66934]: Deploying daemon crash.smithi174 on smithi174 2024-03-26T21:02:44.496 INFO:teuthology.orchestra.run.smithi063.stdout: 5 185884 11.19 MB/sec execute 278 sec latency 56.805 ms 2024-03-26T21:02:44.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:44 smithi174 ceph-mon[53014]: from='client.44125 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:02:44.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:44 smithi174 ceph-mon[53014]: pgmap v44: 97 pgs: 97 active+clean; 354 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 7.6 MiB/s wr, 746 op/s 2024-03-26T21:02:44.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:44 smithi174 ceph-mon[53014]: from='client.34194 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:02:44.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:44 smithi174 ceph-mon[53014]: Upgrade: Updating crash.smithi174 (2/2) 2024-03-26T21:02:44.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:44 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:44.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:44 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi174", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-03-26T21:02:44.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:44 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:02:44.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:44 smithi174 ceph-mon[53014]: Deploying daemon crash.smithi174 on smithi174 2024-03-26T21:02:45.452 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:45 smithi063 ceph-mon[66934]: from='client.? 172.21.15.63:0/2799727625' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-26T21:02:45.452 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:45 smithi063 ceph-mon[66934]: pgmap v45: 97 pgs: 97 active+clean; 354 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.9 MiB/s wr, 485 op/s 2024-03-26T21:02:45.496 INFO:teuthology.orchestra.run.smithi063.stdout: 5 186180 11.16 MB/sec execute 279 sec latency 53.056 ms 2024-03-26T21:02:45.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:45 smithi174 ceph-mon[53014]: from='client.? 172.21.15.63:0/2799727625' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-26T21:02:45.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:45 smithi174 ceph-mon[53014]: pgmap v45: 97 pgs: 97 active+clean; 354 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.9 MiB/s wr, 485 op/s 2024-03-26T21:02:46.496 INFO:teuthology.orchestra.run.smithi063.stdout: 5 186524 11.17 MB/sec execute 280 sec latency 34.111 ms 2024-03-26T21:02:47.496 INFO:teuthology.orchestra.run.smithi063.stdout: 5 187255 11.22 MB/sec execute 281 sec latency 29.181 ms 2024-03-26T21:02:47.970 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:47 smithi063 ceph-mon[66934]: pgmap v46: 97 pgs: 97 active+clean; 349 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 7.5 MiB/s wr, 737 op/s 2024-03-26T21:02:48.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:47 smithi174 ceph-mon[53014]: pgmap v46: 97 pgs: 97 active+clean; 349 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 7.5 MiB/s wr, 737 op/s 2024-03-26T21:02:48.496 INFO:teuthology.orchestra.run.smithi063.stdout: 5 187841 11.22 MB/sec execute 282 sec latency 24.759 ms 2024-03-26T21:02:49.233 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:49 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:49.233 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:49 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:49.233 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:49 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:02:49.432 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:49 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:49.432 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:49 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:49.432 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:49 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:02:49.497 INFO:teuthology.orchestra.run.smithi063.stdout: 5 188318 11.23 MB/sec execute 283 sec latency 29.646 ms 2024-03-26T21:02:50.251 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:50 smithi063 ceph-mon[66934]: pgmap v47: 97 pgs: 97 active+clean; 349 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.1 MiB/s wr, 484 op/s 2024-03-26T21:02:50.496 INFO:teuthology.orchestra.run.smithi063.stdout: 5 188847 11.25 MB/sec execute 284 sec latency 35.846 ms 2024-03-26T21:02:50.502 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:50 smithi174 ceph-mon[53014]: pgmap v47: 97 pgs: 97 active+clean; 349 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.1 MiB/s wr, 484 op/s 2024-03-26T21:02:51.497 INFO:teuthology.orchestra.run.smithi063.stdout: 5 189301 11.25 MB/sec execute 285 sec latency 58.337 ms 2024-03-26T21:02:52.267 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:52 smithi063 ceph-mon[66934]: pgmap v48: 97 pgs: 97 active+clean; 349 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.1 MiB/s wr, 484 op/s 2024-03-26T21:02:52.481 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:52 smithi174 ceph-mon[53014]: pgmap v48: 97 pgs: 97 active+clean; 349 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.1 MiB/s wr, 484 op/s 2024-03-26T21:02:52.496 INFO:teuthology.orchestra.run.smithi063.stdout: 5 189688 11.22 MB/sec execute 286 sec latency 68.788 ms 2024-03-26T21:02:53.496 INFO:teuthology.orchestra.run.smithi063.stdout: 5 190010 11.22 MB/sec execute 287 sec latency 74.556 ms 2024-03-26T21:02:54.061 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:53 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:54.061 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:53 smithi063 ceph-mon[66934]: pgmap v49: 97 pgs: 97 active+clean; 349 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 12 KiB/s rd, 8.3 MiB/s wr, 808 op/s 2024-03-26T21:02:54.061 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:53 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:54.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:53 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:54.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:53 smithi174 ceph-mon[53014]: pgmap v49: 97 pgs: 97 active+clean; 349 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 12 KiB/s rd, 8.3 MiB/s wr, 808 op/s 2024-03-26T21:02:54.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:53 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:54.497 INFO:teuthology.orchestra.run.smithi063.stdout: 5 190186 11.20 MB/sec execute 288 sec latency 74.518 ms 2024-03-26T21:02:55.353 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:55 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:55.353 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:55 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:55.486 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:55 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:55.486 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:55 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:55.496 INFO:teuthology.orchestra.run.smithi063.stdout: 5 190474 11.20 MB/sec execute 289 sec latency 66.465 ms 2024-03-26T21:02:56.374 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:56 smithi063 ceph-mon[66934]: pgmap v50: 97 pgs: 97 active+clean; 349 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.8 MiB/s wr, 575 op/s 2024-03-26T21:02:56.375 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:56 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:02:56.497 INFO:teuthology.orchestra.run.smithi063.stdout: 5 190741 11.20 MB/sec execute 290 sec latency 83.488 ms 2024-03-26T21:02:56.544 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:56 smithi174 ceph-mon[53014]: pgmap v50: 97 pgs: 97 active+clean; 349 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.8 MiB/s wr, 575 op/s 2024-03-26T21:02:56.545 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:56 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:02:57.389 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:57 smithi063 ceph-mon[66934]: mgrmap e42: smithi063.lxhujt(active, since 90s), standbys: smithi174.sexoin 2024-03-26T21:02:57.389 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:57 smithi063 ceph-mon[66934]: pgmap v51: 97 pgs: 97 active+clean; 347 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 7.6 MiB/s wr, 756 op/s 2024-03-26T21:02:57.389 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:57 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:57.389 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:57 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:57.498 INFO:teuthology.orchestra.run.smithi063.stdout: 5 191042 11.17 MB/sec execute 291 sec latency 50.981 ms 2024-03-26T21:02:57.536 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:57 smithi174 ceph-mon[53014]: mgrmap e42: smithi063.lxhujt(active, since 90s), standbys: smithi174.sexoin 2024-03-26T21:02:57.537 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:57 smithi174 ceph-mon[53014]: pgmap v51: 97 pgs: 97 active+clean; 347 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 7.6 MiB/s wr, 756 op/s 2024-03-26T21:02:57.537 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:57 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:57.537 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:57 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:58.497 INFO:teuthology.orchestra.run.smithi063.stdout: 5 191295 11.16 MB/sec execute 292 sec latency 54.744 ms 2024-03-26T21:02:59.436 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:59 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:59.436 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:59 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:59.436 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:59 smithi063 ceph-mon[66934]: pgmap v52: 97 pgs: 97 active+clean; 347 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.0 MiB/s wr, 503 op/s 2024-03-26T21:02:59.436 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:59 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:59.436 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:02:59 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:59.497 INFO:teuthology.orchestra.run.smithi063.stdout: 5 191510 11.13 MB/sec execute 293 sec latency 56.478 ms 2024-03-26T21:02:59.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:59 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:59.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:59 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:59.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:59 smithi174 ceph-mon[53014]: pgmap v52: 97 pgs: 97 active+clean; 347 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.0 MiB/s wr, 503 op/s 2024-03-26T21:02:59.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:59 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:02:59.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:02:59 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:03:00.497 INFO:teuthology.orchestra.run.smithi063.stdout: 5 191780 11.13 MB/sec execute 294 sec latency 40.934 ms 2024-03-26T21:03:01.497 INFO:teuthology.orchestra.run.smithi063.stdout: 5 192066 11.11 MB/sec execute 295 sec latency 50.883 ms 2024-03-26T21:03:01.990 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:01 smithi063 ceph-mon[66934]: pgmap v53: 97 pgs: 97 active+clean; 347 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.0 MiB/s wr, 503 op/s 2024-03-26T21:03:02.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:01 smithi174 ceph-mon[53014]: pgmap v53: 97 pgs: 97 active+clean; 347 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.0 MiB/s wr, 503 op/s 2024-03-26T21:03:02.497 INFO:teuthology.orchestra.run.smithi063.stdout: 5 192469 11.13 MB/sec execute 296 sec latency 41.629 ms 2024-03-26T21:03:03.497 INFO:teuthology.orchestra.run.smithi063.stdout: 5 192893 11.12 MB/sec execute 297 sec latency 47.817 ms 2024-03-26T21:03:04.005 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:03 smithi063 ceph-mon[66934]: pgmap v54: 97 pgs: 97 active+clean; 349 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 10 KiB/s rd, 6.9 MiB/s wr, 671 op/s 2024-03-26T21:03:04.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:03 smithi174 ceph-mon[53014]: pgmap v54: 97 pgs: 97 active+clean; 349 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 10 KiB/s rd, 6.9 MiB/s wr, 671 op/s 2024-03-26T21:03:04.497 INFO:teuthology.orchestra.run.smithi063.stdout: 5 193244 11.09 MB/sec execute 298 sec latency 61.065 ms 2024-03-26T21:03:05.497 INFO:teuthology.orchestra.run.smithi063.stdout: 5 193564 11.09 MB/sec execute 299 sec latency 51.987 ms 2024-03-26T21:03:05.546 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:05 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:03:05.546 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:05 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:03:05.546 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:05 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:03:05.546 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:05 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T21:03:05.546 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:05 smithi063 ceph-mon[66934]: pgmap v55: 97 pgs: 97 active+clean; 349 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.8 MiB/s wr, 362 op/s 2024-03-26T21:03:05.546 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:05 smithi063 ceph-mon[66934]: pgmap v56: 97 pgs: 97 active+clean; 349 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.6 MiB/s wr, 439 op/s 2024-03-26T21:03:05.546 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:05 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:03:05.546 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:05 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:03:05.547 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:05 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:03:05.547 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:05 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:03:05.547 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:05 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:03:05.547 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:05 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:03:05.547 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:05 smithi063 ceph-mon[66934]: Upgrade: Setting container_image for all crash 2024-03-26T21:03:05.547 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:05 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:03:05.547 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:05 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi063"}]: dispatch 2024-03-26T21:03:05.547 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:05 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi063"}]': finished 2024-03-26T21:03:05.547 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:05 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi174"}]: dispatch 2024-03-26T21:03:05.547 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:05 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi174"}]': finished 2024-03-26T21:03:05.547 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:05 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-03-26T21:03:05.547 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:05 smithi063 ceph-mon[66934]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-03-26T21:03:05.548 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:05 smithi063 ceph-mon[66934]: Upgrade: osd.0 is safe to restart 2024-03-26T21:03:05.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:05 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:03:05.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:05 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:03:05.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:05 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:03:05.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:05 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T21:03:05.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:05 smithi174 ceph-mon[53014]: pgmap v55: 97 pgs: 97 active+clean; 349 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.8 MiB/s wr, 362 op/s 2024-03-26T21:03:05.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:05 smithi174 ceph-mon[53014]: pgmap v56: 97 pgs: 97 active+clean; 349 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.6 MiB/s wr, 439 op/s 2024-03-26T21:03:05.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:05 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:03:05.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:05 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:03:05.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:05 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:03:05.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:05 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:03:05.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:05 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:03:05.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:05 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:03:05.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:05 smithi174 ceph-mon[53014]: Upgrade: Setting container_image for all crash 2024-03-26T21:03:05.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:05 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:03:05.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:05 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi063"}]: dispatch 2024-03-26T21:03:05.800 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:05 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi063"}]': finished 2024-03-26T21:03:05.800 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:05 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi174"}]: dispatch 2024-03-26T21:03:05.800 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:05 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi174"}]': finished 2024-03-26T21:03:05.800 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:05 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-03-26T21:03:05.800 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:05 smithi174 ceph-mon[53014]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-03-26T21:03:05.800 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:05 smithi174 ceph-mon[53014]: Upgrade: osd.0 is safe to restart 2024-03-26T21:03:06.498 INFO:teuthology.orchestra.run.smithi063.stdout: 5 193869 11.08 MB/sec execute 300 sec latency 47.777 ms 2024-03-26T21:03:07.498 INFO:teuthology.orchestra.run.smithi063.stdout: 5 194259 11.10 MB/sec execute 301 sec latency 108.902 ms 2024-03-26T21:03:07.921 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:07 smithi063 ceph-mon[66934]: pgmap v57: 97 pgs: 97 active+clean; 349 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 3.6 KiB/s rd, 2.3 MiB/s wr, 211 op/s 2024-03-26T21:03:07.921 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:07 smithi063 ceph-mon[66934]: Upgrade: Updating osd.0 2024-03-26T21:03:07.921 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:07 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:03:07.921 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:07 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-03-26T21:03:07.921 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:07 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:03:07.921 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:07 smithi063 ceph-mon[66934]: Deploying daemon osd.0 on smithi063 2024-03-26T21:03:08.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:07 smithi174 ceph-mon[53014]: pgmap v57: 97 pgs: 97 active+clean; 349 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 3.6 KiB/s rd, 2.3 MiB/s wr, 211 op/s 2024-03-26T21:03:08.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:07 smithi174 ceph-mon[53014]: Upgrade: Updating osd.0 2024-03-26T21:03:08.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:07 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:03:08.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:07 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-03-26T21:03:08.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:07 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:03:08.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:07 smithi174 ceph-mon[53014]: Deploying daemon osd.0 on smithi063 2024-03-26T21:03:08.504 INFO:teuthology.orchestra.run.smithi063.stdout: 5 194758 11.10 MB/sec execute 302 sec latency 32.675 ms 2024-03-26T21:03:09.504 INFO:teuthology.orchestra.run.smithi063.stdout: 5 195219 11.10 MB/sec execute 303 sec latency 31.998 ms 2024-03-26T21:03:09.933 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:09 smithi063 ceph-mon[66934]: pgmap v58: 97 pgs: 97 active+clean; 349 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.1 MiB/s wr, 497 op/s 2024-03-26T21:03:10.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:09 smithi174 ceph-mon[53014]: pgmap v58: 97 pgs: 97 active+clean; 349 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.1 MiB/s wr, 497 op/s 2024-03-26T21:03:10.504 INFO:teuthology.orchestra.run.smithi063.stdout: 5 195689 11.10 MB/sec execute 304 sec latency 37.661 ms 2024-03-26T21:03:10.939 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:10 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:03:11.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:10 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:03:11.504 INFO:teuthology.orchestra.run.smithi063.stdout: 5 196282 11.12 MB/sec execute 305 sec latency 29.786 ms 2024-03-26T21:03:11.954 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:11 smithi063 ceph-mon[66934]: pgmap v59: 97 pgs: 97 active+clean; 349 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.1 MiB/s wr, 497 op/s 2024-03-26T21:03:12.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:11 smithi174 ceph-mon[53014]: pgmap v59: 97 pgs: 97 active+clean; 349 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.1 MiB/s wr, 497 op/s 2024-03-26T21:03:12.504 INFO:teuthology.orchestra.run.smithi063.stdout: 5 196718 11.11 MB/sec execute 306 sec latency 47.038 ms 2024-03-26T21:03:12.978 INFO:journalctl@ceph.osd.0.smithi063.stdout:Mar 26 21:03:12 smithi063 systemd[1]: Stopping Ceph osd.0 for 0ed4dd4c-ebb2-11ee-95d0-87774f69a715... 2024-03-26T21:03:13.226 INFO:journalctl@ceph.osd.0.smithi063.stdout:Mar 26 21:03:13 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-0[35280]: 2024-03-26T21:03:13.172+0000 7faf7f540700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-26T21:03:13.226 INFO:journalctl@ceph.osd.0.smithi063.stdout:Mar 26 21:03:13 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-0[35280]: 2024-03-26T21:03:13.172+0000 7faf7f540700 -1 osd.0 72 *** Got signal Terminated *** 2024-03-26T21:03:13.226 INFO:journalctl@ceph.osd.0.smithi063.stdout:Mar 26 21:03:13 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-0[35280]: 2024-03-26T21:03:13.172+0000 7faf7f540700 -1 osd.0 72 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-26T21:03:13.504 INFO:teuthology.orchestra.run.smithi063.stdout: 5 197057 11.11 MB/sec execute 307 sec latency 84.097 ms 2024-03-26T21:03:14.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:13 smithi063 ceph-mon[66934]: pgmap v60: 97 pgs: 97 active+clean; 349 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 8.2 KiB/s rd, 6.2 MiB/s wr, 645 op/s 2024-03-26T21:03:14.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:13 smithi063 ceph-mon[66934]: osd.0 marked itself down and dead 2024-03-26T21:03:14.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:13 smithi174 ceph-mon[53014]: pgmap v60: 97 pgs: 97 active+clean; 349 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 8.2 KiB/s rd, 6.2 MiB/s wr, 645 op/s 2024-03-26T21:03:14.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:13 smithi174 ceph-mon[53014]: osd.0 marked itself down and dead 2024-03-26T21:03:14.504 INFO:teuthology.orchestra.run.smithi063.stdout: 5 197154 11.08 MB/sec execute 308 sec latency 704.176 ms 2024-03-26T21:03:14.788 INFO:journalctl@ceph.osd.0.smithi063.stdout:Mar 26 21:03:14 smithi063 podman[72494]: 2024-03-26 21:03:14.486754179 +0000 UTC m=+1.447467162 container died 761b1388d8fad80018b60333ac293e65d0d6581f545090c438904d13c795b43a (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-0, distribution-scope=public, io.openshift.expose-services=, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, version=8, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, RELEASE=HEAD, io.buildah.version=1.19.8, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, ceph=True, io.openshift.tags=base centos centos-stream, vcs-type=git, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-03-26T21:03:14.789 INFO:teuthology.orchestra.run.smithi063.stdout:true 2024-03-26T21:03:15.038 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:14 smithi063 ceph-mon[66934]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-26T21:03:15.039 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:14 smithi063 ceph-mon[66934]: osdmap e73: 8 total, 7 up, 8 in 2024-03-26T21:03:15.039 INFO:journalctl@ceph.osd.0.smithi063.stdout:Mar 26 21:03:15 smithi063 podman[72494]: 2024-03-26 21:03:15.0403251 +0000 UTC m=+2.001038076 container cleanup 761b1388d8fad80018b60333ac293e65d0d6581f545090c438904d13c795b43a (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, ceph=True, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, io.openshift.expose-services=, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, maintainer=Guillaume Abrioux , architecture=x86_64, io.buildah.version=1.19.8, GIT_CLEAN=True, com.redhat.component=centos-stream-container) 2024-03-26T21:03:15.039 INFO:journalctl@ceph.osd.0.smithi063.stdout:Mar 26 21:03:15 smithi063 bash[72494]: ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-0 2024-03-26T21:03:15.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:14 smithi174 ceph-mon[53014]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-26T21:03:15.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:14 smithi174 ceph-mon[53014]: osdmap e73: 8 total, 7 up, 8 in 2024-03-26T21:03:15.504 INFO:teuthology.orchestra.run.smithi063.stdout: 5 197431 11.07 MB/sec execute 309 sec latency 1102.963 ms 2024-03-26T21:03:15.546 INFO:journalctl@ceph.osd.0.smithi063.stdout:Mar 26 21:03:15 smithi063 podman[72517]: 2024-03-26 21:03:15.378307547 +0000 UTC m=+0.881761118 container remove 761b1388d8fad80018b60333ac293e65d0d6581f545090c438904d13c795b43a (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, io.buildah.version=1.19.8, GIT_CLEAN=True, release=754, com.redhat.component=centos-stream-container, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, maintainer=Guillaume Abrioux , RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-03-26T21:03:15.869 INFO:teuthology.orchestra.run.smithi063.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-26T21:03:15.869 INFO:teuthology.orchestra.run.smithi063.stdout:alertmanager.smithi063 smithi063 *:9093,9094 running (4m) 18s ago 13m 15.9M - 0.23.0 ba2b418f427c 6e160c94aac3 2024-03-26T21:03:15.870 INFO:teuthology.orchestra.run.smithi063.stdout:crash.smithi063 smithi063 running (34s) 18s ago 13m 6665k - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 cf1096d84270 2024-03-26T21:03:15.870 INFO:teuthology.orchestra.run.smithi063.stdout:crash.smithi174 smithi174 running (28s) 23s ago 12m 6665k - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 cb5371a92684 2024-03-26T21:03:15.870 INFO:teuthology.orchestra.run.smithi063.stdout:grafana.smithi063 smithi063 *:3000 error 18s ago 13m - - 2024-03-26T21:03:15.870 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foofs.smithi063.tzaznz smithi063 running (7m) 18s ago 7m 37.9M - 17.2.0 e1d6a67b021e 10a989fc7954 2024-03-26T21:03:15.870 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foofs.smithi174.gitdvk smithi174 running (7m) 22s ago 7m 13.4M - 17.2.0 e1d6a67b021e 4339a5d98d1d 2024-03-26T21:03:15.870 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.smithi063.lxhujt smithi063 *:8443,9283,8765 running (3m) 18s ago 14m 501M - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 6d9d10330a90 2024-03-26T21:03:15.870 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.smithi174.sexoin smithi174 *:8443,9283,8765 running (2m) 22s ago 12m 450M - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 d7546233672d 2024-03-26T21:03:15.870 INFO:teuthology.orchestra.run.smithi063.stdout:mon.smithi063 smithi063 running (2m) 18s ago 15m 45.3M 2048M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 c644200344f8 2024-03-26T21:03:15.870 INFO:teuthology.orchestra.run.smithi063.stdout:mon.smithi174 smithi174 running (92s) 22s ago 12m 39.0M 2048M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 1a6d442692e7 2024-03-26T21:03:15.870 INFO:teuthology.orchestra.run.smithi063.stdout:nfs.foo.0.0.smithi063.fgmpwa smithi063 *:2049 running (7m) 18s ago 7m 274M - 4.0 e1d6a67b021e 3d6c0dd9ce51 2024-03-26T21:03:15.870 INFO:teuthology.orchestra.run.smithi063.stdout:nfs.foo.1.0.smithi174.pytptt smithi174 *:2049 running (7m) 22s ago 7m 86.4M - 4.0 e1d6a67b021e f220dbb449a3 2024-03-26T21:03:15.870 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.smithi063 smithi063 *:9100 running (4m) 18s ago 13m 12.3M - 1.3.1 1dbe0e931976 f52f41fd14c0 2024-03-26T21:03:15.870 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.smithi174 smithi174 *:9100 running (3m) 22s ago 12m 17.2M - 1.3.1 1dbe0e931976 0c027f39a4c8 2024-03-26T21:03:15.870 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0 smithi063 running (12m) 18s ago 12m 390M 1936M 17.2.0 e1d6a67b021e 761b1388d8fa 2024-03-26T21:03:15.871 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1 smithi063 running (11m) 18s ago 11m 569M 1936M 17.2.0 e1d6a67b021e 108275c17cad 2024-03-26T21:03:15.871 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2 smithi063 running (11m) 18s ago 11m 436M 1936M 17.2.0 e1d6a67b021e 107f685ad08d 2024-03-26T21:03:15.871 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3 smithi063 running (10m) 18s ago 10m 590M 1936M 17.2.0 e1d6a67b021e ab0f1a855957 2024-03-26T21:03:15.871 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4 smithi174 running (10m) 22s ago 10m 575M 2704M 17.2.0 e1d6a67b021e a0e877ecc06d 2024-03-26T21:03:15.871 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5 smithi174 running (9m) 22s ago 9m 423M 2704M 17.2.0 e1d6a67b021e 378c2aca7e71 2024-03-26T21:03:15.871 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6 smithi174 running (9m) 22s ago 9m 328M 2704M 17.2.0 e1d6a67b021e ffbb5b7ad3b6 2024-03-26T21:03:15.871 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7 smithi174 running (9m) 22s ago 9m 524M 2704M 17.2.0 e1d6a67b021e c1d57fcabb3f 2024-03-26T21:03:15.871 INFO:teuthology.orchestra.run.smithi063.stdout:prometheus.smithi063 smithi063 *:9095 running (2m) 18s ago 12m 65.1M - 2.33.4 514e6a882f6e 567eef202627 2024-03-26T21:03:16.054 INFO:journalctl@ceph.osd.0.smithi063.stdout:Mar 26 21:03:16 smithi063 podman[72663]: 2024-03-26 21:03:15.929609407 +0000 UTC m=+0.028648159 image pull a9bdc3eaeb106993bcbbac35d7fe5c7b76c052021676e129f5984aed30aee063 quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d 2024-03-26T21:03:16.055 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:15 smithi063 ceph-mon[66934]: pgmap v62: 97 pgs: 13 stale+active+clean, 84 active+clean; 349 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.9 MiB/s wr, 614 op/s 2024-03-26T21:03:16.055 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:15 smithi063 ceph-mon[66934]: from='client.34202 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:03:16.055 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:15 smithi063 ceph-mon[66934]: osdmap e74: 8 total, 7 up, 8 in 2024-03-26T21:03:16.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:15 smithi174 ceph-mon[53014]: pgmap v62: 97 pgs: 13 stale+active+clean, 84 active+clean; 349 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.9 MiB/s wr, 614 op/s 2024-03-26T21:03:16.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:15 smithi174 ceph-mon[53014]: from='client.34202 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:03:16.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:15 smithi174 ceph-mon[53014]: osdmap e74: 8 total, 7 up, 8 in 2024-03-26T21:03:16.305 INFO:journalctl@ceph.osd.0.smithi063.stdout:Mar 26 21:03:16 smithi063 podman[72663]: 2024-03-26 21:03:16.06141479 +0000 UTC m=+0.160453532 container create 9051b922a2b7dff35342291fcb4902674ad3016fee3df8c228b6fccd7fb0db94 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-0-deactivate, CEPH_POINT_RELEASE=, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, io.buildah.version=1.33.5, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-03-26T21:03:16.450 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-03-26T21:03:16.450 INFO:teuthology.orchestra.run.smithi063.stdout: "mon": { 2024-03-26T21:03:16.450 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 2 2024-03-26T21:03:16.450 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:03:16.451 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr": { 2024-03-26T21:03:16.451 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 2 2024-03-26T21:03:16.451 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:03:16.451 INFO:teuthology.orchestra.run.smithi063.stdout: "osd": { 2024-03-26T21:03:16.451 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7 2024-03-26T21:03:16.451 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:03:16.451 INFO:teuthology.orchestra.run.smithi063.stdout: "mds": { 2024-03-26T21:03:16.451 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-26T21:03:16.451 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:03:16.451 INFO:teuthology.orchestra.run.smithi063.stdout: "overall": { 2024-03-26T21:03:16.451 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 9, 2024-03-26T21:03:16.451 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 4 2024-03-26T21:03:16.451 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-03-26T21:03:16.451 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-03-26T21:03:16.504 INFO:teuthology.orchestra.run.smithi063.stdout: 5 197947 11.10 MB/sec execute 310 sec latency 37.484 ms 2024-03-26T21:03:16.814 INFO:journalctl@ceph.osd.0.smithi063.stdout:Mar 26 21:03:16 smithi063 podman[72663]: 2024-03-26 21:03:16.666087276 +0000 UTC m=+0.765126045 container init 9051b922a2b7dff35342291fcb4902674ad3016fee3df8c228b6fccd7fb0db94 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-0-deactivate, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-03-26T21:03:16.814 INFO:journalctl@ceph.osd.0.smithi063.stdout:Mar 26 21:03:16 smithi063 podman[72663]: 2024-03-26 21:03:16.679091106 +0000 UTC m=+0.778129852 container start 9051b922a2b7dff35342291fcb4902674ad3016fee3df8c228b6fccd7fb0db94 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-0-deactivate, org.label-schema.build-date=20240326, ceph=True, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-03-26T21:03:16.931 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-03-26T21:03:16.932 INFO:teuthology.orchestra.run.smithi063.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d", 2024-03-26T21:03:16.932 INFO:teuthology.orchestra.run.smithi063.stdout: "in_progress": true, 2024-03-26T21:03:16.932 INFO:teuthology.orchestra.run.smithi063.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-26T21:03:16.932 INFO:teuthology.orchestra.run.smithi063.stdout: "services_complete": [ 2024-03-26T21:03:16.932 INFO:teuthology.orchestra.run.smithi063.stdout: "crash", 2024-03-26T21:03:16.932 INFO:teuthology.orchestra.run.smithi063.stdout: "mon", 2024-03-26T21:03:16.932 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr" 2024-03-26T21:03:16.932 INFO:teuthology.orchestra.run.smithi063.stdout: ], 2024-03-26T21:03:16.932 INFO:teuthology.orchestra.run.smithi063.stdout: "progress": "6/23 daemons upgraded", 2024-03-26T21:03:16.932 INFO:teuthology.orchestra.run.smithi063.stdout: "message": "Currently upgrading osd daemons", 2024-03-26T21:03:16.932 INFO:teuthology.orchestra.run.smithi063.stdout: "is_paused": false 2024-03-26T21:03:16.932 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-03-26T21:03:17.065 INFO:journalctl@ceph.osd.0.smithi063.stdout:Mar 26 21:03:16 smithi063 podman[72663]: 2024-03-26 21:03:16.908523557 +0000 UTC m=+1.007562301 container attach 9051b922a2b7dff35342291fcb4902674ad3016fee3df8c228b6fccd7fb0db94 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-0-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, org.label-schema.build-date=20240326, GIT_CLEAN=True, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-03-26T21:03:17.065 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:16 smithi063 ceph-mon[66934]: from='client.34206 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:03:17.065 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:16 smithi063 ceph-mon[66934]: from='client.44147 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:03:17.065 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:16 smithi063 ceph-mon[66934]: from='client.? 172.21.15.63:0/1582303348' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:03:17.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:16 smithi174 ceph-mon[53014]: from='client.34206 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:03:17.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:16 smithi174 ceph-mon[53014]: from='client.44147 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:03:17.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:16 smithi174 ceph-mon[53014]: from='client.? 172.21.15.63:0/1582303348' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:03:17.315 INFO:journalctl@ceph.osd.0.smithi063.stdout:Mar 26 21:03:17 smithi063 podman[72663]: 2024-03-26 21:03:17.13885595 +0000 UTC m=+1.237894690 container died 9051b922a2b7dff35342291fcb4902674ad3016fee3df8c228b6fccd7fb0db94 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-0-deactivate, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240326, GIT_CLEAN=True) 2024-03-26T21:03:17.504 INFO:teuthology.orchestra.run.smithi063.stdout: 5 198358 11.09 MB/sec execute 311 sec latency 33.009 ms 2024-03-26T21:03:17.646 INFO:teuthology.orchestra.run.smithi063.stdout:HEALTH_WARN 1 failed cephadm daemon(s); 1 osds down 2024-03-26T21:03:17.646 INFO:teuthology.orchestra.run.smithi063.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-26T21:03:17.646 INFO:teuthology.orchestra.run.smithi063.stdout: daemon grafana.smithi063 on smithi063 is in error state 2024-03-26T21:03:17.646 INFO:teuthology.orchestra.run.smithi063.stdout:[WRN] OSD_DOWN: 1 osds down 2024-03-26T21:03:17.647 INFO:teuthology.orchestra.run.smithi063.stdout: osd.0 (root=default,host=smithi063) is down 2024-03-26T21:03:17.838 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:17 smithi063 ceph-mon[66934]: pgmap v64: 97 pgs: 13 stale+active+clean, 84 active+clean; 349 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 4.9 KiB/s rd, 4.2 MiB/s wr, 427 op/s 2024-03-26T21:03:17.838 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:17 smithi063 ceph-mon[66934]: from='client.34218 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:03:17.838 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:17 smithi063 ceph-mon[66934]: from='client.? 172.21.15.63:0/896848052' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-26T21:03:18.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:17 smithi174 ceph-mon[53014]: pgmap v64: 97 pgs: 13 stale+active+clean, 84 active+clean; 349 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 4.9 KiB/s rd, 4.2 MiB/s wr, 427 op/s 2024-03-26T21:03:18.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:17 smithi174 ceph-mon[53014]: from='client.34218 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:03:18.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:17 smithi174 ceph-mon[53014]: from='client.? 172.21.15.63:0/896848052' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-26T21:03:18.340 INFO:journalctl@ceph.osd.0.smithi063.stdout:Mar 26 21:03:18 smithi063 podman[72758]: 2024-03-26 21:03:18.195580088 +0000 UTC m=+1.050704623 container remove 9051b922a2b7dff35342291fcb4902674ad3016fee3df8c228b6fccd7fb0db94 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-0-deactivate, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, org.label-schema.schema-version=1.0, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.license=GPLv2) 2024-03-26T21:03:18.340 INFO:journalctl@ceph.osd.0.smithi063.stdout:Mar 26 21:03:18 smithi063 systemd[1]: ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715@osd.0.service: Deactivated successfully. 2024-03-26T21:03:18.340 INFO:journalctl@ceph.osd.0.smithi063.stdout:Mar 26 21:03:18 smithi063 systemd[1]: Stopped Ceph osd.0 for 0ed4dd4c-ebb2-11ee-95d0-87774f69a715. 2024-03-26T21:03:18.340 INFO:journalctl@ceph.osd.0.smithi063.stdout:Mar 26 21:03:18 smithi063 systemd[1]: ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715@osd.0.service: Consumed 3min 36.423s CPU time. 2024-03-26T21:03:18.504 INFO:teuthology.orchestra.run.smithi063.stdout: 5 198696 11.08 MB/sec execute 312 sec latency 54.231 ms 2024-03-26T21:03:18.590 INFO:journalctl@ceph.osd.0.smithi063.stdout:Mar 26 21:03:18 smithi063 systemd[1]: Starting Ceph osd.0 for 0ed4dd4c-ebb2-11ee-95d0-87774f69a715... 2024-03-26T21:03:18.864 INFO:journalctl@ceph.osd.0.smithi063.stdout:Mar 26 21:03:18 smithi063 podman[72895]: 2024-03-26 21:03:18.670343661 +0000 UTC m=+0.022597007 image pull a9bdc3eaeb106993bcbbac35d7fe5c7b76c052021676e129f5984aed30aee063 quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d 2024-03-26T21:03:18.864 INFO:journalctl@ceph.osd.0.smithi063.stdout:Mar 26 21:03:18 smithi063 podman[72895]: 2024-03-26 21:03:18.777359899 +0000 UTC m=+0.129613295 container create 994573f4b56378a2af069ad751eca5c4f561801d52b0b9a68d1806aea5e8a660 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-0-activate, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-03-26T21:03:19.115 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:18 smithi063 ceph-mon[66934]: Health check failed: Degraded data redundancy: 223/1674 objects degraded (13.321%), 29 pgs degraded (PG_DEGRADED) 2024-03-26T21:03:19.116 INFO:journalctl@ceph.osd.0.smithi063.stdout:Mar 26 21:03:19 smithi063 podman[72895]: 2024-03-26 21:03:19.055626573 +0000 UTC m=+0.407879918 container init 994573f4b56378a2af069ad751eca5c4f561801d52b0b9a68d1806aea5e8a660 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-0-activate, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240326, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-03-26T21:03:19.116 INFO:journalctl@ceph.osd.0.smithi063.stdout:Mar 26 21:03:19 smithi063 podman[72895]: 2024-03-26 21:03:19.06108446 +0000 UTC m=+0.413337797 container start 994573f4b56378a2af069ad751eca5c4f561801d52b0b9a68d1806aea5e8a660 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-0-activate, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240326, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5) 2024-03-26T21:03:19.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:18 smithi174 ceph-mon[53014]: Health check failed: Degraded data redundancy: 223/1674 objects degraded (13.321%), 29 pgs degraded (PG_DEGRADED) 2024-03-26T21:03:19.371 INFO:journalctl@ceph.osd.0.smithi063.stdout:Mar 26 21:03:19 smithi063 podman[72895]: 2024-03-26 21:03:19.180869528 +0000 UTC m=+0.533122871 container attach 994573f4b56378a2af069ad751eca5c4f561801d52b0b9a68d1806aea5e8a660 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-0-activate, org.label-schema.vendor=CentOS, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-03-26T21:03:19.504 INFO:teuthology.orchestra.run.smithi063.stdout: 5 199069 11.08 MB/sec execute 313 sec latency 42.745 ms 2024-03-26T21:03:19.873 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:19 smithi063 ceph-mon[66934]: pgmap v65: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 353 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 9.6 KiB/s rd, 7.4 MiB/s wr, 730 op/s; 223/1674 objects degraded (13.321%); 0 B/s, 0 objects/s recovering 2024-03-26T21:03:19.874 INFO:journalctl@ceph.osd.0.smithi063.stdout:Mar 26 21:03:19 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-0-activate[72909]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-03-26T21:03:19.874 INFO:journalctl@ceph.osd.0.smithi063.stdout:Mar 26 21:03:19 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-0-activate[72909]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-03-26T21:03:19.874 INFO:journalctl@ceph.osd.0.smithi063.stdout:Mar 26 21:03:19 smithi063 bash[72895]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-03-26T21:03:19.874 INFO:journalctl@ceph.osd.0.smithi063.stdout:Mar 26 21:03:19 smithi063 bash[72895]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-03-26T21:03:19.874 INFO:journalctl@ceph.osd.0.smithi063.stdout:Mar 26 21:03:19 smithi063 bash[72895]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-03-26T21:03:19.874 INFO:journalctl@ceph.osd.0.smithi063.stdout:Mar 26 21:03:19 smithi063 bash[72895]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-03-26T21:03:19.875 INFO:journalctl@ceph.osd.0.smithi063.stdout:Mar 26 21:03:19 smithi063 bash[72895]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-03-26T21:03:19.875 INFO:journalctl@ceph.osd.0.smithi063.stdout:Mar 26 21:03:19 smithi063 bash[72895]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-03-26T21:03:19.875 INFO:journalctl@ceph.osd.0.smithi063.stdout:Mar 26 21:03:19 smithi063 bash[72895]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-03-26T21:03:19.875 INFO:journalctl@ceph.osd.0.smithi063.stdout:Mar 26 21:03:19 smithi063 bash[72895]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-03-26T21:03:19.875 INFO:journalctl@ceph.osd.0.smithi063.stdout:Mar 26 21:03:19 smithi063 bash[72895]: --> ceph-volume raw activate successful for osd ID: 0 2024-03-26T21:03:19.875 INFO:journalctl@ceph.osd.0.smithi063.stdout:Mar 26 21:03:19 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-0-activate[72909]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-03-26T21:03:19.875 INFO:journalctl@ceph.osd.0.smithi063.stdout:Mar 26 21:03:19 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-0-activate[72909]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-03-26T21:03:19.875 INFO:journalctl@ceph.osd.0.smithi063.stdout:Mar 26 21:03:19 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-0-activate[72909]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-03-26T21:03:19.875 INFO:journalctl@ceph.osd.0.smithi063.stdout:Mar 26 21:03:19 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-0-activate[72909]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-03-26T21:03:19.876 INFO:journalctl@ceph.osd.0.smithi063.stdout:Mar 26 21:03:19 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-0-activate[72909]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-03-26T21:03:19.876 INFO:journalctl@ceph.osd.0.smithi063.stdout:Mar 26 21:03:19 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-0-activate[72909]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-03-26T21:03:19.876 INFO:journalctl@ceph.osd.0.smithi063.stdout:Mar 26 21:03:19 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-0-activate[72909]: --> ceph-volume raw activate successful for osd ID: 0 2024-03-26T21:03:19.876 INFO:journalctl@ceph.osd.0.smithi063.stdout:Mar 26 21:03:19 smithi063 podman[72895]: 2024-03-26 21:03:19.749307279 +0000 UTC m=+1.101560614 container died 994573f4b56378a2af069ad751eca5c4f561801d52b0b9a68d1806aea5e8a660 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-0-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-03-26T21:03:20.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:19 smithi174 ceph-mon[53014]: pgmap v65: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 353 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 9.6 KiB/s rd, 7.4 MiB/s wr, 730 op/s; 223/1674 objects degraded (13.321%); 0 B/s, 0 objects/s recovering 2024-03-26T21:03:20.504 INFO:teuthology.orchestra.run.smithi063.stdout: 5 199509 11.09 MB/sec execute 314 sec latency 35.123 ms 2024-03-26T21:03:20.895 INFO:journalctl@ceph.osd.0.smithi063.stdout:Mar 26 21:03:20 smithi063 podman[73035]: 2024-03-26 21:03:20.771992047 +0000 UTC m=+1.008732721 container remove 994573f4b56378a2af069ad751eca5c4f561801d52b0b9a68d1806aea5e8a660 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-0-activate, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, ceph=True, CEPH_POINT_RELEASE=, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS) 2024-03-26T21:03:21.150 INFO:journalctl@ceph.osd.0.smithi063.stdout:Mar 26 21:03:21 smithi063 podman[73088]: 2024-03-26 21:03:20.941020983 +0000 UTC m=+0.023201272 image pull a9bdc3eaeb106993bcbbac35d7fe5c7b76c052021676e129f5984aed30aee063 quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d 2024-03-26T21:03:21.150 INFO:journalctl@ceph.osd.0.smithi063.stdout:Mar 26 21:03:21 smithi063 podman[73088]: 2024-03-26 21:03:21.056443661 +0000 UTC m=+0.138623947 container create 2d4b3fa825cf4317bccdaaad523d19f04ef839b6ee325cb0068c769c15fe11da (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-0, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240326, maintainer=Guillaume Abrioux ) 2024-03-26T21:03:21.504 INFO:teuthology.orchestra.run.smithi063.stdout: 5 199927 11.08 MB/sec execute 315 sec latency 51.587 ms 2024-03-26T21:03:21.649 INFO:journalctl@ceph.osd.0.smithi063.stdout:Mar 26 21:03:21 smithi063 podman[73088]: 2024-03-26 21:03:21.407272492 +0000 UTC m=+0.489452795 container init 2d4b3fa825cf4317bccdaaad523d19f04ef839b6ee325cb0068c769c15fe11da (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-0, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, ceph=True, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=) 2024-03-26T21:03:21.649 INFO:journalctl@ceph.osd.0.smithi063.stdout:Mar 26 21:03:21 smithi063 podman[73088]: 2024-03-26 21:03:21.411731142 +0000 UTC m=+0.493911438 container start 2d4b3fa825cf4317bccdaaad523d19f04ef839b6ee325cb0068c769c15fe11da (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-0, io.buildah.version=1.33.5, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD) 2024-03-26T21:03:21.649 INFO:journalctl@ceph.osd.0.smithi063.stdout:Mar 26 21:03:21 smithi063 ceph-osd[73106]: -- 172.21.15.63:0/1504817812 <== mon.0 v2:172.21.15.63:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x56532157b680 con 0x565321544c00 2024-03-26T21:03:21.649 INFO:journalctl@ceph.osd.0.smithi063.stdout:Mar 26 21:03:21 smithi063 bash[73088]: 2d4b3fa825cf4317bccdaaad523d19f04ef839b6ee325cb0068c769c15fe11da 2024-03-26T21:03:21.649 INFO:journalctl@ceph.osd.0.smithi063.stdout:Mar 26 21:03:21 smithi063 systemd[1]: Started Ceph osd.0 for 0ed4dd4c-ebb2-11ee-95d0-87774f69a715. 2024-03-26T21:03:21.908 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:21 smithi063 ceph-mon[66934]: pgmap v66: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 353 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.2 MiB/s wr, 303 op/s; 223/1674 objects degraded (13.321%); 0 B/s, 0 objects/s recovering 2024-03-26T21:03:21.909 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:21 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:03:21.909 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:21 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:03:21.909 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:21 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:03:22.161 INFO:journalctl@ceph.osd.0.smithi063.stdout:Mar 26 21:03:21 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-0[73102]: 2024-03-26T21:03:21.993+0000 7f4221cd07c0 -1 Falling back to public interface 2024-03-26T21:03:22.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:21 smithi174 ceph-mon[53014]: pgmap v66: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 353 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.2 MiB/s wr, 303 op/s; 223/1674 objects degraded (13.321%); 0 B/s, 0 objects/s recovering 2024-03-26T21:03:22.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:21 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:03:22.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:21 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:03:22.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:21 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:03:22.504 INFO:teuthology.orchestra.run.smithi063.stdout: 5 200222 11.06 MB/sec execute 316 sec latency 55.186 ms 2024-03-26T21:03:23.504 INFO:teuthology.orchestra.run.smithi063.stdout: 5 200536 11.05 MB/sec execute 317 sec latency 47.110 ms 2024-03-26T21:03:23.960 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:23 smithi063 ceph-mon[66934]: pgmap v67: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 347 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 9.6 KiB/s rd, 6.3 MiB/s wr, 620 op/s; 218/1638 objects degraded (13.309%); 0 B/s, 0 objects/s recovering 2024-03-26T21:03:24.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:23 smithi174 ceph-mon[53014]: pgmap v67: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 347 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 9.6 KiB/s rd, 6.3 MiB/s wr, 620 op/s; 218/1638 objects degraded (13.309%); 0 B/s, 0 objects/s recovering 2024-03-26T21:03:24.504 INFO:teuthology.orchestra.run.smithi063.stdout: 5 200784 11.04 MB/sec execute 318 sec latency 52.243 ms 2024-03-26T21:03:24.970 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:24 smithi063 ceph-mon[66934]: Health check update: Degraded data redundancy: 218/1638 objects degraded (13.309%), 29 pgs degraded (PG_DEGRADED) 2024-03-26T21:03:25.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:24 smithi174 ceph-mon[53014]: Health check update: Degraded data redundancy: 218/1638 objects degraded (13.309%), 29 pgs degraded (PG_DEGRADED) 2024-03-26T21:03:25.504 INFO:teuthology.orchestra.run.smithi063.stdout: 5 201112 11.05 MB/sec execute 319 sec latency 59.065 ms 2024-03-26T21:03:25.974 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:25 smithi063 ceph-mon[66934]: pgmap v68: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 347 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.4 MiB/s wr, 532 op/s; 218/1638 objects degraded (13.309%); 0 B/s, 0 objects/s recovering 2024-03-26T21:03:25.975 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:25 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:03:25.975 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:25 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:03:26.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:25 smithi174 ceph-mon[53014]: pgmap v68: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 347 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.4 MiB/s wr, 532 op/s; 218/1638 objects degraded (13.309%); 0 B/s, 0 objects/s recovering 2024-03-26T21:03:26.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:25 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:03:26.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:25 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:03:26.505 INFO:teuthology.orchestra.run.smithi063.stdout: 5 201781 11.07 MB/sec execute 320 sec latency 50.910 ms 2024-03-26T21:03:27.505 INFO:teuthology.orchestra.run.smithi063.stdout: 5 202370 11.08 MB/sec execute 321 sec latency 21.663 ms 2024-03-26T21:03:28.006 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:27 smithi063 ceph-mon[66934]: pgmap v69: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 347 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.7 MiB/s wr, 459 op/s; 218/1638 objects degraded (13.309%); 0 B/s, 0 objects/s recovering 2024-03-26T21:03:28.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:27 smithi174 ceph-mon[53014]: pgmap v69: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 347 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.7 MiB/s wr, 459 op/s; 218/1638 objects degraded (13.309%); 0 B/s, 0 objects/s recovering 2024-03-26T21:03:28.505 INFO:teuthology.orchestra.run.smithi063.stdout: 5 203106 11.12 MB/sec execute 322 sec latency 33.972 ms 2024-03-26T21:03:29.505 INFO:teuthology.orchestra.run.smithi063.stdout: 5 203580 11.11 MB/sec execute 323 sec latency 42.287 ms 2024-03-26T21:03:29.767 INFO:journalctl@ceph.osd.0.smithi063.stdout:Mar 26 21:03:29 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-0[73102]: 2024-03-26T21:03:29.672+0000 7f4221cd07c0 -1 osd.0 0 read_superblock omap replica is missing. 2024-03-26T21:03:30.021 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:29 smithi063 ceph-mon[66934]: pgmap v70: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 347 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.8 MiB/s wr, 669 op/s; 223/1695 objects degraded (13.156%); 0 B/s, 0 objects/s recovering 2024-03-26T21:03:30.021 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:29 smithi063 ceph-mon[66934]: Health check update: Degraded data redundancy: 223/1695 objects degraded (13.156%), 29 pgs degraded (PG_DEGRADED) 2024-03-26T21:03:30.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:29 smithi174 ceph-mon[53014]: pgmap v70: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 347 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.8 MiB/s wr, 669 op/s; 223/1695 objects degraded (13.156%); 0 B/s, 0 objects/s recovering 2024-03-26T21:03:30.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:29 smithi174 ceph-mon[53014]: Health check update: Degraded data redundancy: 223/1695 objects degraded (13.156%), 29 pgs degraded (PG_DEGRADED) 2024-03-26T21:03:30.505 INFO:teuthology.orchestra.run.smithi063.stdout: 5 203889 11.09 MB/sec execute 324 sec latency 37.756 ms 2024-03-26T21:03:30.989 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:30 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:03:30.989 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:30 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:03:31.294 INFO:journalctl@ceph.osd.0.smithi063.stdout:Mar 26 21:03:31 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-0[73102]: 2024-03-26T21:03:31.228+0000 7f4221cd07c0 -1 osd.0 72 log_to_monitors true 2024-03-26T21:03:31.297 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:30 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:03:31.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:30 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:03:31.505 INFO:teuthology.orchestra.run.smithi063.stdout: 5 204232 11.09 MB/sec execute 325 sec latency 54.699 ms 2024-03-26T21:03:32.067 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:31 smithi063 ceph-mon[66934]: pgmap v71: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 347 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.7 MiB/s wr, 466 op/s; 223/1695 objects degraded (13.156%) 2024-03-26T21:03:32.067 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:31 smithi063 ceph-mon[66934]: from='osd.0 [v2:172.21.15.63:6802/364802273,v1:172.21.15.63:6803/364802273]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-03-26T21:03:32.067 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:31 smithi063 ceph-mon[66934]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-03-26T21:03:32.067 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:31 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:03:32.067 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:31 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:03:32.067 INFO:journalctl@ceph.osd.0.smithi063.stdout:Mar 26 21:03:32 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-0[73102]: 2024-03-26T21:03:32.038+0000 7f4213128700 -1 osd.0 72 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-26T21:03:32.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:31 smithi174 ceph-mon[53014]: pgmap v71: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 347 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.7 MiB/s wr, 466 op/s; 223/1695 objects degraded (13.156%) 2024-03-26T21:03:32.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:31 smithi174 ceph-mon[53014]: from='osd.0 [v2:172.21.15.63:6802/364802273,v1:172.21.15.63:6803/364802273]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-03-26T21:03:32.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:31 smithi174 ceph-mon[53014]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-03-26T21:03:32.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:31 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:03:32.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:31 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:03:32.505 INFO:teuthology.orchestra.run.smithi063.stdout: 5 204670 11.11 MB/sec execute 326 sec latency 37.516 ms 2024-03-26T21:03:33.077 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:33 smithi063 ceph-mon[66934]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-03-26T21:03:33.078 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:33 smithi063 ceph-mon[66934]: osdmap e75: 8 total, 7 up, 8 in 2024-03-26T21:03:33.078 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:33 smithi063 ceph-mon[66934]: from='osd.0 [v2:172.21.15.63:6802/364802273,v1:172.21.15.63:6803/364802273]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi063", "root=default"]}]: dispatch 2024-03-26T21:03:33.078 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:33 smithi063 ceph-mon[66934]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi063", "root=default"]}]: dispatch 2024-03-26T21:03:33.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:33 smithi174 ceph-mon[53014]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-03-26T21:03:33.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:33 smithi174 ceph-mon[53014]: osdmap e75: 8 total, 7 up, 8 in 2024-03-26T21:03:33.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:33 smithi174 ceph-mon[53014]: from='osd.0 [v2:172.21.15.63:6802/364802273,v1:172.21.15.63:6803/364802273]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi063", "root=default"]}]: dispatch 2024-03-26T21:03:33.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:33 smithi174 ceph-mon[53014]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi063", "root=default"]}]: dispatch 2024-03-26T21:03:33.505 INFO:teuthology.orchestra.run.smithi063.stdout: 5 205020 11.10 MB/sec execute 327 sec latency 496.010 ms 2024-03-26T21:03:34.225 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:34 smithi063 ceph-mon[66934]: pgmap v73: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 347 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 9.0 KiB/s rd, 6.4 MiB/s wr, 633 op/s; 224/1677 objects degraded (13.357%) 2024-03-26T21:03:34.225 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:34 smithi063 ceph-mon[66934]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-26T21:03:34.225 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:34 smithi063 ceph-mon[66934]: osd.0 [v2:172.21.15.63:6802/364802273,v1:172.21.15.63:6803/364802273] boot 2024-03-26T21:03:34.225 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:34 smithi063 ceph-mon[66934]: osdmap e76: 8 total, 8 up, 8 in 2024-03-26T21:03:34.225 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:34 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-26T21:03:34.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:34 smithi174 ceph-mon[53014]: pgmap v73: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 347 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 9.0 KiB/s rd, 6.4 MiB/s wr, 633 op/s; 224/1677 objects degraded (13.357%) 2024-03-26T21:03:34.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:34 smithi174 ceph-mon[53014]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-26T21:03:34.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:34 smithi174 ceph-mon[53014]: osd.0 [v2:172.21.15.63:6802/364802273,v1:172.21.15.63:6803/364802273] boot 2024-03-26T21:03:34.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:34 smithi174 ceph-mon[53014]: osdmap e76: 8 total, 8 up, 8 in 2024-03-26T21:03:34.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:34 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-26T21:03:34.505 INFO:teuthology.orchestra.run.smithi063.stdout: 5 205020 11.07 MB/sec execute 328 sec latency 1496.070 ms 2024-03-26T21:03:35.278 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:35 smithi063 ceph-mon[66934]: osdmap e77: 8 total, 8 up, 8 in 2024-03-26T21:03:35.278 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:35 smithi063 ceph-mon[66934]: Health check update: Degraded data redundancy: 224/1677 objects degraded (13.357%), 29 pgs degraded (PG_DEGRADED) 2024-03-26T21:03:35.278 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:35 smithi063 ceph-mon[66934]: osdmap e78: 8 total, 8 up, 8 in 2024-03-26T21:03:35.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:35 smithi174 ceph-mon[53014]: osdmap e77: 8 total, 8 up, 8 in 2024-03-26T21:03:35.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:35 smithi174 ceph-mon[53014]: Health check update: Degraded data redundancy: 224/1677 objects degraded (13.357%), 29 pgs degraded (PG_DEGRADED) 2024-03-26T21:03:35.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:35 smithi174 ceph-mon[53014]: osdmap e78: 8 total, 8 up, 8 in 2024-03-26T21:03:35.505 INFO:teuthology.orchestra.run.smithi063.stdout: 5 205185 11.04 MB/sec execute 329 sec latency 2064.318 ms 2024-03-26T21:03:36.037 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:36 smithi063 ceph-mon[66934]: pgmap v76: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 347 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 9.3 KiB/s rd, 6.2 MiB/s wr, 605 op/s; 224/1677 objects degraded (13.357%) 2024-03-26T21:03:36.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:36 smithi174 ceph-mon[53014]: pgmap v76: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 347 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 9.3 KiB/s rd, 6.2 MiB/s wr, 605 op/s; 224/1677 objects degraded (13.357%) 2024-03-26T21:03:36.505 INFO:teuthology.orchestra.run.smithi063.stdout: 5 205628 11.03 MB/sec execute 330 sec latency 460.107 ms 2024-03-26T21:03:37.066 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:37 smithi063 ceph-mon[66934]: osdmap e79: 8 total, 8 up, 8 in 2024-03-26T21:03:37.066 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:37 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:03:37.066 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:37 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:03:37.066 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:37 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:03:37.066 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:37 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T21:03:37.066 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:37 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:03:37.066 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:37 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:03:37.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:37 smithi174 ceph-mon[53014]: osdmap e79: 8 total, 8 up, 8 in 2024-03-26T21:03:37.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:37 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:03:37.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:37 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:03:37.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:37 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:03:37.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:37 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T21:03:37.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:37 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:03:37.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:37 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:03:37.505 INFO:teuthology.orchestra.run.smithi063.stdout: 5 205935 11.03 MB/sec execute 331 sec latency 1011.765 ms 2024-03-26T21:03:38.071 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:38 smithi063 ceph-mon[66934]: pgmap v79: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 347 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 224/1677 objects degraded (13.357%) 2024-03-26T21:03:38.071 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:38 smithi063 ceph-mon[66934]: pgmap v80: 97 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 4 peering, 3 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+undersized, 77 active+clean; 343 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.1 KiB/s rd, 5.0 MiB/s wr, 510 op/s; 234/1668 objects degraded (14.029%); 2.1 MiB/s, 30 objects/s recovering 2024-03-26T21:03:38.071 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:38 smithi063 ceph-mon[66934]: pgmap v81: 97 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 4 peering, 3 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+undersized, 77 active+clean; 343 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 8.2 KiB/s rd, 6.7 MiB/s wr, 685 op/s; 234/1668 objects degraded (14.029%); 2.8 MiB/s, 40 objects/s recovering 2024-03-26T21:03:38.071 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:38 smithi063 ceph-mon[66934]: osdmap e80: 8 total, 8 up, 8 in 2024-03-26T21:03:38.071 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:38 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:03:38.071 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:38 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:03:38.071 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:38 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:03:38.071 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:38 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:03:38.071 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:38 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-26T21:03:38.072 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:38 smithi063 ceph-mon[66934]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-26T21:03:38.072 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:38 smithi063 ceph-mon[66934]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-03-26T21:03:38.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:38 smithi174 ceph-mon[53014]: pgmap v79: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 347 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 224/1677 objects degraded (13.357%) 2024-03-26T21:03:38.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:38 smithi174 ceph-mon[53014]: pgmap v80: 97 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 4 peering, 3 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+undersized, 77 active+clean; 343 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.1 KiB/s rd, 5.0 MiB/s wr, 510 op/s; 234/1668 objects degraded (14.029%); 2.1 MiB/s, 30 objects/s recovering 2024-03-26T21:03:38.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:38 smithi174 ceph-mon[53014]: pgmap v81: 97 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 4 peering, 3 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+undersized, 77 active+clean; 343 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 8.2 KiB/s rd, 6.7 MiB/s wr, 685 op/s; 234/1668 objects degraded (14.029%); 2.8 MiB/s, 40 objects/s recovering 2024-03-26T21:03:38.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:38 smithi174 ceph-mon[53014]: osdmap e80: 8 total, 8 up, 8 in 2024-03-26T21:03:38.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:38 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:03:38.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:38 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:03:38.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:38 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:03:38.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:38 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:03:38.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:38 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-26T21:03:38.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:38 smithi174 ceph-mon[53014]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-26T21:03:38.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:38 smithi174 ceph-mon[53014]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-03-26T21:03:38.505 INFO:teuthology.orchestra.run.smithi063.stdout: 5 206369 11.04 MB/sec execute 332 sec latency 345.129 ms 2024-03-26T21:03:39.101 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:39 smithi063 ceph-mon[66934]: osdmap e81: 8 total, 8 up, 8 in 2024-03-26T21:03:39.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:39 smithi174 ceph-mon[53014]: osdmap e81: 8 total, 8 up, 8 in 2024-03-26T21:03:39.508 INFO:teuthology.orchestra.run.smithi063.stdout: 5 206613 11.03 MB/sec execute 333 sec latency 910.298 ms 2024-03-26T21:03:40.110 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:40 smithi063 ceph-mon[66934]: pgmap v84: 97 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 6 peering, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 342 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 8.3 KiB/s rd, 6.8 MiB/s wr, 694 op/s; 209/1662 objects degraded (12.575%); 2.8 MiB/s, 49 objects/s recovering 2024-03-26T21:03:40.111 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:40 smithi063 ceph-mon[66934]: osdmap e82: 8 total, 8 up, 8 in 2024-03-26T21:03:40.111 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:40 smithi063 ceph-mon[66934]: Health check update: Degraded data redundancy: 209/1662 objects degraded (12.575%), 8 pgs degraded (PG_DEGRADED) 2024-03-26T21:03:40.508 INFO:teuthology.orchestra.run.smithi063.stdout: 5 207069 11.03 MB/sec execute 334 sec latency 44.236 ms 2024-03-26T21:03:40.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:40 smithi174 ceph-mon[53014]: pgmap v84: 97 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 6 peering, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 342 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 8.3 KiB/s rd, 6.8 MiB/s wr, 694 op/s; 209/1662 objects degraded (12.575%); 2.8 MiB/s, 49 objects/s recovering 2024-03-26T21:03:40.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:40 smithi174 ceph-mon[53014]: osdmap e82: 8 total, 8 up, 8 in 2024-03-26T21:03:40.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:40 smithi174 ceph-mon[53014]: Health check update: Degraded data redundancy: 209/1662 objects degraded (12.575%), 8 pgs degraded (PG_DEGRADED) 2024-03-26T21:03:41.115 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:41 smithi063 ceph-mon[66934]: osdmap e83: 8 total, 8 up, 8 in 2024-03-26T21:03:41.115 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:41 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:03:41.116 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:41 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:03:41.508 INFO:teuthology.orchestra.run.smithi063.stdout: 5 207477 11.01 MB/sec execute 335 sec latency 235.724 ms 2024-03-26T21:03:41.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:41 smithi174 ceph-mon[53014]: osdmap e83: 8 total, 8 up, 8 in 2024-03-26T21:03:41.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:41 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:03:41.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:41 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:03:42.132 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:42 smithi063 ceph-mon[66934]: pgmap v87: 97 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 6 peering, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 342 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.0 KiB/s wr, 1 op/s; 209/1662 objects degraded (12.575%); 0 B/s, 5 objects/s recovering 2024-03-26T21:03:42.133 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:42 smithi063 ceph-mon[66934]: osdmap e84: 8 total, 8 up, 8 in 2024-03-26T21:03:42.508 INFO:teuthology.orchestra.run.smithi063.stdout: 5 207892 11.02 MB/sec execute 336 sec latency 808.351 ms 2024-03-26T21:03:42.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:42 smithi174 ceph-mon[53014]: pgmap v87: 97 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 6 peering, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 342 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.0 KiB/s wr, 1 op/s; 209/1662 objects degraded (12.575%); 0 B/s, 5 objects/s recovering 2024-03-26T21:03:42.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:42 smithi174 ceph-mon[53014]: osdmap e84: 8 total, 8 up, 8 in 2024-03-26T21:03:43.194 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:43 smithi063 ceph-mon[66934]: osdmap e85: 8 total, 8 up, 8 in 2024-03-26T21:03:43.194 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:43 smithi063 ceph-mon[66934]: osdmap e86: 8 total, 8 up, 8 in 2024-03-26T21:03:43.508 INFO:teuthology.orchestra.run.smithi063.stdout: 5 208132 11.02 MB/sec execute 337 sec latency 70.755 ms 2024-03-26T21:03:43.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:43 smithi174 ceph-mon[53014]: osdmap e85: 8 total, 8 up, 8 in 2024-03-26T21:03:43.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:43 smithi174 ceph-mon[53014]: osdmap e86: 8 total, 8 up, 8 in 2024-03-26T21:03:44.222 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:44 smithi063 ceph-mon[66934]: pgmap v90: 97 pgs: 1 active+undersized+remapped, 1 peering, 95 active+clean; 340 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 10 KiB/s rd, 7.0 MiB/s wr, 698 op/s; 532 KiB/s, 53 objects/s recovering 2024-03-26T21:03:44.222 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:44 smithi063 ceph-mon[66934]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 209/1662 objects degraded (12.575%), 8 pgs degraded) 2024-03-26T21:03:44.508 INFO:teuthology.orchestra.run.smithi063.stdout: 5 208484 11.01 MB/sec execute 338 sec latency 53.864 ms 2024-03-26T21:03:44.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:44 smithi174 ceph-mon[53014]: pgmap v90: 97 pgs: 1 active+undersized+remapped, 1 peering, 95 active+clean; 340 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 10 KiB/s rd, 7.0 MiB/s wr, 698 op/s; 532 KiB/s, 53 objects/s recovering 2024-03-26T21:03:44.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:44 smithi174 ceph-mon[53014]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 209/1662 objects degraded (12.575%), 8 pgs degraded) 2024-03-26T21:03:45.508 INFO:teuthology.orchestra.run.smithi063.stdout: 5 208728 11.01 MB/sec execute 339 sec latency 101.092 ms 2024-03-26T21:03:46.292 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:46 smithi063 ceph-mon[66934]: pgmap v92: 97 pgs: 1 active+undersized+remapped, 1 peering, 95 active+clean; 340 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.8 MiB/s wr, 573 op/s; 436 KiB/s, 43 objects/s recovering 2024-03-26T21:03:46.508 INFO:teuthology.orchestra.run.smithi063.stdout: 5 208937 10.99 MB/sec execute 340 sec latency 44.802 ms 2024-03-26T21:03:46.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:46 smithi174 ceph-mon[53014]: pgmap v92: 97 pgs: 1 active+undersized+remapped, 1 peering, 95 active+clean; 340 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.8 MiB/s wr, 573 op/s; 436 KiB/s, 43 objects/s recovering 2024-03-26T21:03:47.508 INFO:teuthology.orchestra.run.smithi063.stdout: 5 209213 10.97 MB/sec execute 341 sec latency 35.431 ms 2024-03-26T21:03:48.192 INFO:teuthology.orchestra.run.smithi063.stdout:true 2024-03-26T21:03:48.330 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:48 smithi063 ceph-mon[66934]: pgmap v93: 97 pgs: 97 active+clean; 338 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 12 KiB/s rd, 7.8 MiB/s wr, 795 op/s; 354 KiB/s, 35 objects/s recovering 2024-03-26T21:03:48.508 INFO:teuthology.orchestra.run.smithi063.stdout: 5 209446 10.95 MB/sec execute 342 sec latency 48.240 ms 2024-03-26T21:03:48.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:48 smithi174 ceph-mon[53014]: pgmap v93: 97 pgs: 97 active+clean; 338 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 12 KiB/s rd, 7.8 MiB/s wr, 795 op/s; 354 KiB/s, 35 objects/s recovering 2024-03-26T21:03:49.117 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:49 smithi063 ceph-mon[66934]: from='client.44167 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:03:49.208 INFO:teuthology.orchestra.run.smithi063.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-26T21:03:49.208 INFO:teuthology.orchestra.run.smithi063.stdout:alertmanager.smithi063 smithi063 *:9093,9094 running (5m) 19s ago 14m 16.0M - 0.23.0 ba2b418f427c 6e160c94aac3 2024-03-26T21:03:49.208 INFO:teuthology.orchestra.run.smithi063.stdout:crash.smithi063 smithi063 running (67s) 19s ago 14m 6665k - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 cf1096d84270 2024-03-26T21:03:49.208 INFO:teuthology.orchestra.run.smithi063.stdout:crash.smithi174 smithi174 running (61s) 56s ago 13m 6665k - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 cb5371a92684 2024-03-26T21:03:49.208 INFO:teuthology.orchestra.run.smithi063.stdout:grafana.smithi063 smithi063 *:3000 error 19s ago 13m - - 2024-03-26T21:03:49.208 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foofs.smithi063.tzaznz smithi063 running (8m) 19s ago 8m 38.0M - 17.2.0 e1d6a67b021e 10a989fc7954 2024-03-26T21:03:49.208 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foofs.smithi174.gitdvk smithi174 running (8m) 56s ago 8m 13.4M - 17.2.0 e1d6a67b021e 4339a5d98d1d 2024-03-26T21:03:49.208 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.smithi063.lxhujt smithi063 *:8443,9283,8765 running (4m) 19s ago 15m 503M - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 6d9d10330a90 2024-03-26T21:03:49.209 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.smithi174.sexoin smithi174 *:8443,9283,8765 running (3m) 56s ago 13m 450M - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 d7546233672d 2024-03-26T21:03:49.209 INFO:teuthology.orchestra.run.smithi063.stdout:mon.smithi063 smithi063 running (2m) 19s ago 15m 48.3M 2048M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 c644200344f8 2024-03-26T21:03:49.209 INFO:teuthology.orchestra.run.smithi063.stdout:mon.smithi174 smithi174 running (2m) 56s ago 13m 39.0M 2048M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 1a6d442692e7 2024-03-26T21:03:49.209 INFO:teuthology.orchestra.run.smithi063.stdout:nfs.foo.0.0.smithi063.fgmpwa smithi063 *:2049 running (7m) 19s ago 7m 274M - 4.0 e1d6a67b021e 3d6c0dd9ce51 2024-03-26T21:03:49.209 INFO:teuthology.orchestra.run.smithi063.stdout:nfs.foo.1.0.smithi174.pytptt smithi174 *:2049 running (7m) 56s ago 7m 86.4M - 4.0 e1d6a67b021e f220dbb449a3 2024-03-26T21:03:49.209 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.smithi063 smithi063 *:9100 running (4m) 19s ago 13m 9420k - 1.3.1 1dbe0e931976 f52f41fd14c0 2024-03-26T21:03:49.209 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.smithi174 smithi174 *:9100 running (4m) 56s ago 13m 17.2M - 1.3.1 1dbe0e931976 0c027f39a4c8 2024-03-26T21:03:49.209 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0 smithi063 running (28s) 19s ago 12m 12.7M 1936M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 2d4b3fa825cf 2024-03-26T21:03:49.209 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1 smithi063 running (12m) 19s ago 12m 343M 1936M 17.2.0 e1d6a67b021e 108275c17cad 2024-03-26T21:03:49.209 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2 smithi063 running (11m) 19s ago 11m 464M 1936M 17.2.0 e1d6a67b021e 107f685ad08d 2024-03-26T21:03:49.209 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3 smithi063 running (11m) 19s ago 11m 387M 1936M 17.2.0 e1d6a67b021e ab0f1a855957 2024-03-26T21:03:49.209 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4 smithi174 running (10m) 56s ago 10m 575M 2704M 17.2.0 e1d6a67b021e a0e877ecc06d 2024-03-26T21:03:49.209 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5 smithi174 running (10m) 56s ago 10m 423M 2704M 17.2.0 e1d6a67b021e 378c2aca7e71 2024-03-26T21:03:49.209 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6 smithi174 running (10m) 56s ago 10m 328M 2704M 17.2.0 e1d6a67b021e ffbb5b7ad3b6 2024-03-26T21:03:49.209 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7 smithi174 running (9m) 56s ago 9m 524M 2704M 17.2.0 e1d6a67b021e c1d57fcabb3f 2024-03-26T21:03:49.210 INFO:teuthology.orchestra.run.smithi063.stdout:prometheus.smithi063 smithi063 *:9095 running (3m) 19s ago 13m 65.2M - 2.33.4 514e6a882f6e 567eef202627 2024-03-26T21:03:49.508 INFO:teuthology.orchestra.run.smithi063.stdout: 5 209682 10.95 MB/sec execute 343 sec latency 60.175 ms 2024-03-26T21:03:49.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:49 smithi174 ceph-mon[53014]: from='client.44167 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:03:49.815 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-03-26T21:03:49.815 INFO:teuthology.orchestra.run.smithi063.stdout: "mon": { 2024-03-26T21:03:49.815 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 2 2024-03-26T21:03:49.816 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:03:49.816 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr": { 2024-03-26T21:03:49.816 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 2 2024-03-26T21:03:49.816 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:03:49.816 INFO:teuthology.orchestra.run.smithi063.stdout: "osd": { 2024-03-26T21:03:49.816 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-03-26T21:03:49.816 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 1 2024-03-26T21:03:49.816 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:03:49.816 INFO:teuthology.orchestra.run.smithi063.stdout: "mds": { 2024-03-26T21:03:49.816 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-26T21:03:49.816 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:03:49.816 INFO:teuthology.orchestra.run.smithi063.stdout: "overall": { 2024-03-26T21:03:49.816 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 9, 2024-03-26T21:03:49.816 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 5 2024-03-26T21:03:49.817 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-03-26T21:03:49.817 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-03-26T21:03:50.144 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:50 smithi063 ceph-mon[66934]: from='client.44171 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:03:50.145 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:50 smithi063 ceph-mon[66934]: pgmap v94: 97 pgs: 97 active+clean; 337 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 9.4 KiB/s rd, 6.4 MiB/s wr, 626 op/s; 270 KiB/s, 27 objects/s recovering 2024-03-26T21:03:50.145 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:50 smithi063 ceph-mon[66934]: from='client.34236 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:03:50.145 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:50 smithi063 ceph-mon[66934]: from='client.? 172.21.15.63:0/1423303063' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:03:50.338 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-03-26T21:03:50.338 INFO:teuthology.orchestra.run.smithi063.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d", 2024-03-26T21:03:50.338 INFO:teuthology.orchestra.run.smithi063.stdout: "in_progress": true, 2024-03-26T21:03:50.338 INFO:teuthology.orchestra.run.smithi063.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-26T21:03:50.338 INFO:teuthology.orchestra.run.smithi063.stdout: "services_complete": [ 2024-03-26T21:03:50.338 INFO:teuthology.orchestra.run.smithi063.stdout: "crash", 2024-03-26T21:03:50.338 INFO:teuthology.orchestra.run.smithi063.stdout: "mon", 2024-03-26T21:03:50.338 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr" 2024-03-26T21:03:50.338 INFO:teuthology.orchestra.run.smithi063.stdout: ], 2024-03-26T21:03:50.339 INFO:teuthology.orchestra.run.smithi063.stdout: "progress": "7/23 daemons upgraded", 2024-03-26T21:03:50.339 INFO:teuthology.orchestra.run.smithi063.stdout: "message": "Currently upgrading osd daemons", 2024-03-26T21:03:50.339 INFO:teuthology.orchestra.run.smithi063.stdout: "is_paused": false 2024-03-26T21:03:50.339 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-03-26T21:03:50.508 INFO:teuthology.orchestra.run.smithi063.stdout: 5 209936 10.94 MB/sec execute 344 sec latency 66.119 ms 2024-03-26T21:03:50.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:50 smithi174 ceph-mon[53014]: from='client.44171 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:03:50.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:50 smithi174 ceph-mon[53014]: pgmap v94: 97 pgs: 97 active+clean; 337 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 9.4 KiB/s rd, 6.4 MiB/s wr, 626 op/s; 270 KiB/s, 27 objects/s recovering 2024-03-26T21:03:50.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:50 smithi174 ceph-mon[53014]: from='client.34236 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:03:50.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:50 smithi174 ceph-mon[53014]: from='client.? 172.21.15.63:0/1423303063' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:03:50.965 INFO:teuthology.orchestra.run.smithi063.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-26T21:03:50.965 INFO:teuthology.orchestra.run.smithi063.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-26T21:03:50.966 INFO:teuthology.orchestra.run.smithi063.stdout: daemon grafana.smithi063 on smithi063 is in error state 2024-03-26T21:03:51.180 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:51 smithi063 ceph-mon[66934]: from='client.? 172.21.15.63:0/1744913367' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-26T21:03:51.508 INFO:teuthology.orchestra.run.smithi063.stdout: 5 210295 10.94 MB/sec execute 345 sec latency 38.620 ms 2024-03-26T21:03:51.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:51 smithi174 ceph-mon[53014]: from='client.? 172.21.15.63:0/1744913367' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-26T21:03:52.201 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:52 smithi063 ceph-mon[66934]: from='client.34244 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:03:52.201 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:52 smithi063 ceph-mon[66934]: pgmap v95: 97 pgs: 97 active+clean; 337 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 3.8 KiB/s rd, 2.5 MiB/s wr, 240 op/s 2024-03-26T21:03:52.508 INFO:teuthology.orchestra.run.smithi063.stdout: 5 210641 10.92 MB/sec execute 346 sec latency 54.089 ms 2024-03-26T21:03:52.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:52 smithi174 ceph-mon[53014]: from='client.34244 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:03:52.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:52 smithi174 ceph-mon[53014]: pgmap v95: 97 pgs: 97 active+clean; 337 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 3.8 KiB/s rd, 2.5 MiB/s wr, 240 op/s 2024-03-26T21:03:53.221 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:53 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-26T21:03:53.221 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:53 smithi063 ceph-mon[66934]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-26T21:03:53.221 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:53 smithi063 ceph-mon[66934]: Upgrade: osd.1 is safe to restart 2024-03-26T21:03:53.508 INFO:teuthology.orchestra.run.smithi063.stdout: 5 210984 10.90 MB/sec execute 347 sec latency 41.950 ms 2024-03-26T21:03:53.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:53 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-26T21:03:53.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:53 smithi174 ceph-mon[53014]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-26T21:03:53.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:53 smithi174 ceph-mon[53014]: Upgrade: osd.1 is safe to restart 2024-03-26T21:03:54.231 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:54 smithi063 ceph-mon[66934]: pgmap v96: 97 pgs: 97 active+clean; 336 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.3 MiB/s wr, 424 op/s 2024-03-26T21:03:54.508 INFO:teuthology.orchestra.run.smithi063.stdout: 5 211258 10.90 MB/sec execute 348 sec latency 60.421 ms 2024-03-26T21:03:54.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:54 smithi174 ceph-mon[53014]: pgmap v96: 97 pgs: 97 active+clean; 336 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.3 MiB/s wr, 424 op/s 2024-03-26T21:03:55.508 INFO:teuthology.orchestra.run.smithi063.stdout: 5 211573 10.90 MB/sec execute 349 sec latency 64.519 ms 2024-03-26T21:03:56.064 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:55 smithi063 ceph-mon[66934]: pgmap v97: 97 pgs: 97 active+clean; 336 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.1 KiB/s rd, 3.6 MiB/s wr, 357 op/s 2024-03-26T21:03:56.064 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:55 smithi063 ceph-mon[66934]: Upgrade: Updating osd.1 2024-03-26T21:03:56.064 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:55 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:03:56.064 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:55 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-03-26T21:03:56.064 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:55 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:03:56.064 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:55 smithi063 ceph-mon[66934]: Deploying daemon osd.1 on smithi063 2024-03-26T21:03:56.064 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:55 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:03:56.064 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:55 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:03:56.064 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:55 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:03:56.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:55 smithi174 ceph-mon[53014]: pgmap v97: 97 pgs: 97 active+clean; 336 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.1 KiB/s rd, 3.6 MiB/s wr, 357 op/s 2024-03-26T21:03:56.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:55 smithi174 ceph-mon[53014]: Upgrade: Updating osd.1 2024-03-26T21:03:56.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:55 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:03:56.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:55 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-03-26T21:03:56.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:55 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:03:56.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:55 smithi174 ceph-mon[53014]: Deploying daemon osd.1 on smithi063 2024-03-26T21:03:56.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:55 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:03:56.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:55 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:03:56.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:55 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:03:56.508 INFO:teuthology.orchestra.run.smithi063.stdout: 5 211893 10.90 MB/sec execute 350 sec latency 45.504 ms 2024-03-26T21:03:57.508 INFO:teuthology.orchestra.run.smithi063.stdout: 5 212237 10.89 MB/sec execute 351 sec latency 71.878 ms 2024-03-26T21:03:58.125 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:03:58 smithi063 ceph-mon[66934]: pgmap v98: 97 pgs: 97 active+clean; 338 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.2 MiB/s wr, 536 op/s 2024-03-26T21:03:58.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:03:58 smithi174 ceph-mon[53014]: pgmap v98: 97 pgs: 97 active+clean; 338 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.2 MiB/s wr, 536 op/s 2024-03-26T21:03:58.509 INFO:teuthology.orchestra.run.smithi063.stdout: 5 212526 10.88 MB/sec execute 352 sec latency 43.197 ms 2024-03-26T21:03:59.509 INFO:teuthology.orchestra.run.smithi063.stdout: 5 212813 10.88 MB/sec execute 353 sec latency 64.383 ms 2024-03-26T21:03:59.917 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 21:03:59 smithi063 systemd[1]: Stopping Ceph osd.1 for 0ed4dd4c-ebb2-11ee-95d0-87774f69a715... 2024-03-26T21:04:00.028 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:00 smithi063 ceph-mon[66934]: pgmap v99: 97 pgs: 97 active+clean; 339 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.0 MiB/s wr, 386 op/s 2024-03-26T21:04:00.073 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 21:04:00 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-1[39331]: 2024-03-26T21:04:00.061+0000 7fbfde35e700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-26T21:04:00.074 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 21:04:00 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-1[39331]: 2024-03-26T21:04:00.061+0000 7fbfde35e700 -1 osd.1 86 *** Got signal Terminated *** 2024-03-26T21:04:00.074 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 21:04:00 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-1[39331]: 2024-03-26T21:04:00.061+0000 7fbfde35e700 -1 osd.1 86 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-26T21:04:00.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:00 smithi174 ceph-mon[53014]: pgmap v99: 97 pgs: 97 active+clean; 339 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.0 MiB/s wr, 386 op/s 2024-03-26T21:04:00.509 INFO:teuthology.orchestra.run.smithi063.stdout: 5 213165 10.87 MB/sec execute 354 sec latency 57.605 ms 2024-03-26T21:04:01.030 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:01 smithi063 ceph-mon[66934]: osd.1 marked itself down and dead 2024-03-26T21:04:01.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:01 smithi174 ceph-mon[53014]: osd.1 marked itself down and dead 2024-03-26T21:04:01.509 INFO:teuthology.orchestra.run.smithi063.stdout: 5 213420 10.86 MB/sec execute 355 sec latency 353.958 ms 2024-03-26T21:04:02.013 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 21:04:01 smithi063 podman[75189]: 2024-03-26 21:04:01.653127276 +0000 UTC m=+1.764583121 container died 108275c17cadb7e7815aabc796c7feb99436de6fac52414a147d82b465d6dd78 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-1, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, name=centos-stream, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, vcs-type=git, io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, version=8, release=754) 2024-03-26T21:04:02.267 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:02 smithi063 ceph-mon[66934]: pgmap v100: 97 pgs: 97 active+clean; 339 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.7 MiB/s wr, 373 op/s 2024-03-26T21:04:02.267 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:02 smithi063 ceph-mon[66934]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-26T21:04:02.267 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:02 smithi063 ceph-mon[66934]: osdmap e87: 8 total, 7 up, 8 in 2024-03-26T21:04:02.267 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 21:04:02 smithi063 podman[75189]: 2024-03-26 21:04:02.235646213 +0000 UTC m=+2.347102058 container cleanup 108275c17cadb7e7815aabc796c7feb99436de6fac52414a147d82b465d6dd78 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-1, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=HEAD, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, vcs-type=git, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, name=centos-stream, GIT_BRANCH=HEAD, io.buildah.version=1.19.8) 2024-03-26T21:04:02.267 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 21:04:02 smithi063 bash[75189]: ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-1 2024-03-26T21:04:02.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:02 smithi174 ceph-mon[53014]: pgmap v100: 97 pgs: 97 active+clean; 339 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.7 MiB/s wr, 373 op/s 2024-03-26T21:04:02.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:02 smithi174 ceph-mon[53014]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-26T21:04:02.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:02 smithi174 ceph-mon[53014]: osdmap e87: 8 total, 7 up, 8 in 2024-03-26T21:04:02.509 INFO:teuthology.orchestra.run.smithi063.stdout: 5 213578 10.85 MB/sec execute 356 sec latency 890.878 ms 2024-03-26T21:04:02.776 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 21:04:02 smithi063 podman[75207]: 2024-03-26 21:04:02.547800219 +0000 UTC m=+0.892140491 container remove 108275c17cadb7e7815aabc796c7feb99436de6fac52414a147d82b465d6dd78 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-1, name=centos-stream, vcs-type=git, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, release=754, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, ceph=True, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , io.openshift.expose-services=, GIT_BRANCH=HEAD, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-03-26T21:04:03.026 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:03 smithi063 ceph-mon[66934]: osdmap e88: 8 total, 7 up, 8 in 2024-03-26T21:04:03.027 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 21:04:02 smithi063 podman[75293]: 2024-03-26 21:04:02.848459305 +0000 UTC m=+0.024368714 image pull a9bdc3eaeb106993bcbbac35d7fe5c7b76c052021676e129f5984aed30aee063 quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d 2024-03-26T21:04:03.277 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 21:04:03 smithi063 podman[75293]: 2024-03-26 21:04:03.185133159 +0000 UTC m=+0.361042557 container create 4c8142fd7b8e07f748781cecd515cbe2c4de72ff4e84f9e64f12377ed26e7b21 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-1-deactivate, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, GIT_BRANCH=HEAD, org.label-schema.build-date=20240326, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux ) 2024-03-26T21:04:03.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:03 smithi174 ceph-mon[53014]: osdmap e88: 8 total, 7 up, 8 in 2024-03-26T21:04:03.509 INFO:teuthology.orchestra.run.smithi063.stdout: 5 213993 10.85 MB/sec execute 357 sec latency 38.374 ms 2024-03-26T21:04:03.779 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 21:04:03 smithi063 podman[75293]: 2024-03-26 21:04:03.61860809 +0000 UTC m=+0.794517489 container init 4c8142fd7b8e07f748781cecd515cbe2c4de72ff4e84f9e64f12377ed26e7b21 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-1-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, GIT_CLEAN=True) 2024-03-26T21:04:03.780 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 21:04:03 smithi063 podman[75293]: 2024-03-26 21:04:03.629094709 +0000 UTC m=+0.805004105 container start 4c8142fd7b8e07f748781cecd515cbe2c4de72ff4e84f9e64f12377ed26e7b21 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-1-deactivate, org.label-schema.schema-version=1.0, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-26T21:04:04.031 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:04 smithi063 ceph-mon[66934]: pgmap v103: 97 pgs: 3 activating+undersized, 1 active+undersized+degraded, 35 peering, 58 active+clean; 344 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.7 MiB/s wr, 565 op/s; 15/1647 objects degraded (0.911%) 2024-03-26T21:04:04.032 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:04 smithi063 ceph-mon[66934]: Health check failed: Reduced data availability: 4 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2024-03-26T21:04:04.032 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:04 smithi063 ceph-mon[66934]: Health check failed: Degraded data redundancy: 15/1647 objects degraded (0.911%), 1 pg degraded (PG_DEGRADED) 2024-03-26T21:04:04.032 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 21:04:03 smithi063 podman[75293]: 2024-03-26 21:04:03.788389844 +0000 UTC m=+0.964299240 container attach 4c8142fd7b8e07f748781cecd515cbe2c4de72ff4e84f9e64f12377ed26e7b21 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-1-deactivate, org.label-schema.schema-version=1.0, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, org.label-schema.vendor=CentOS, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux ) 2024-03-26T21:04:04.298 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 21:04:04 smithi063 podman[75293]: 2024-03-26 21:04:04.100911971 +0000 UTC m=+1.276821365 container died 4c8142fd7b8e07f748781cecd515cbe2c4de72ff4e84f9e64f12377ed26e7b21 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-1-deactivate, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326, io.buildah.version=1.33.5, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, ceph=True, org.label-schema.schema-version=1.0, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-03-26T21:04:04.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:04 smithi174 ceph-mon[53014]: pgmap v103: 97 pgs: 3 activating+undersized, 1 active+undersized+degraded, 35 peering, 58 active+clean; 344 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.7 MiB/s wr, 565 op/s; 15/1647 objects degraded (0.911%) 2024-03-26T21:04:04.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:04 smithi174 ceph-mon[53014]: Health check failed: Reduced data availability: 4 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2024-03-26T21:04:04.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:04 smithi174 ceph-mon[53014]: Health check failed: Degraded data redundancy: 15/1647 objects degraded (0.911%), 1 pg degraded (PG_DEGRADED) 2024-03-26T21:04:04.509 INFO:teuthology.orchestra.run.smithi063.stdout: 5 214399 10.83 MB/sec execute 358 sec latency 37.885 ms 2024-03-26T21:04:05.509 INFO:teuthology.orchestra.run.smithi063.stdout: 5 214768 10.82 MB/sec execute 359 sec latency 41.679 ms 2024-03-26T21:04:05.557 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 21:04:05 smithi063 podman[75319]: 2024-03-26 21:04:05.479999843 +0000 UTC m=+1.372025533 container remove 4c8142fd7b8e07f748781cecd515cbe2c4de72ff4e84f9e64f12377ed26e7b21 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-1-deactivate, GIT_BRANCH=HEAD, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e) 2024-03-26T21:04:05.558 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 21:04:05 smithi063 systemd[1]: ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715@osd.1.service: Deactivated successfully. 2024-03-26T21:04:05.558 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 21:04:05 smithi063 systemd[1]: Stopped Ceph osd.1 for 0ed4dd4c-ebb2-11ee-95d0-87774f69a715. 2024-03-26T21:04:05.558 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 21:04:05 smithi063 systemd[1]: ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715@osd.1.service: Consumed 3min 54.340s CPU time. 2024-03-26T21:04:06.068 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:06 smithi063 ceph-mon[66934]: pgmap v104: 97 pgs: 3 activating+undersized, 1 active+undersized+degraded, 35 peering, 58 active+clean; 344 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 4.4 KiB/s rd, 3.3 MiB/s wr, 291 op/s; 15/1647 objects degraded (0.911%) 2024-03-26T21:04:06.068 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 21:04:05 smithi063 systemd[1]: Starting Ceph osd.1 for 0ed4dd4c-ebb2-11ee-95d0-87774f69a715... 2024-03-26T21:04:06.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:06 smithi174 ceph-mon[53014]: pgmap v104: 97 pgs: 3 activating+undersized, 1 active+undersized+degraded, 35 peering, 58 active+clean; 344 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 4.4 KiB/s rd, 3.3 MiB/s wr, 291 op/s; 15/1647 objects degraded (0.911%) 2024-03-26T21:04:06.318 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 21:04:06 smithi063 podman[75415]: 2024-03-26 21:04:06.038998281 +0000 UTC m=+0.025992159 image pull a9bdc3eaeb106993bcbbac35d7fe5c7b76c052021676e129f5984aed30aee063 quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d 2024-03-26T21:04:06.509 INFO:teuthology.orchestra.run.smithi063.stdout: 5 215239 10.84 MB/sec execute 360 sec latency 45.409 ms 2024-03-26T21:04:06.572 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 21:04:06 smithi063 podman[75415]: 2024-03-26 21:04:06.339990804 +0000 UTC m=+0.326984732 container create 27d2f283b70a06aaa26570990b194c021f6faa1e33d1c34a51353dc998e7df59 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-1-activate, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, io.buildah.version=1.33.5) 2024-03-26T21:04:06.823 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 21:04:06 smithi063 podman[75415]: 2024-03-26 21:04:06.760934858 +0000 UTC m=+0.747928750 container init 27d2f283b70a06aaa26570990b194c021f6faa1e33d1c34a51353dc998e7df59 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-1-activate, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, org.label-schema.schema-version=1.0, ceph=True) 2024-03-26T21:04:06.823 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 21:04:06 smithi063 podman[75415]: 2024-03-26 21:04:06.766692209 +0000 UTC m=+0.753686095 container start 27d2f283b70a06aaa26570990b194c021f6faa1e33d1c34a51353dc998e7df59 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-1-activate, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, CEPH_POINT_RELEASE=, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-26T21:04:07.076 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 21:04:06 smithi063 podman[75415]: 2024-03-26 21:04:06.973373914 +0000 UTC m=+0.960367793 container attach 27d2f283b70a06aaa26570990b194c021f6faa1e33d1c34a51353dc998e7df59 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-1-activate, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, CEPH_POINT_RELEASE=, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-03-26T21:04:07.330 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:07 smithi063 ceph-mon[66934]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 4 pgs peering) 2024-03-26T21:04:07.509 INFO:teuthology.orchestra.run.smithi063.stdout: 5 215743 10.85 MB/sec execute 361 sec latency 37.019 ms 2024-03-26T21:04:07.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:07 smithi174 ceph-mon[53014]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 4 pgs peering) 2024-03-26T21:04:07.585 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 21:04:07 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-1-activate[75429]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-03-26T21:04:07.585 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 21:04:07 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-1-activate[75429]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-03-26T21:04:07.586 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 21:04:07 smithi063 bash[75415]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-03-26T21:04:07.586 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 21:04:07 smithi063 bash[75415]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-03-26T21:04:07.586 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 21:04:07 smithi063 bash[75415]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-03-26T21:04:07.586 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 21:04:07 smithi063 bash[75415]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-03-26T21:04:07.586 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 21:04:07 smithi063 bash[75415]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-03-26T21:04:07.586 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 21:04:07 smithi063 bash[75415]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-03-26T21:04:07.586 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 21:04:07 smithi063 bash[75415]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-03-26T21:04:07.586 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 21:04:07 smithi063 bash[75415]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-03-26T21:04:07.586 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 21:04:07 smithi063 bash[75415]: --> ceph-volume raw activate successful for osd ID: 1 2024-03-26T21:04:07.586 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 21:04:07 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-1-activate[75429]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-03-26T21:04:07.586 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 21:04:07 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-1-activate[75429]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-03-26T21:04:07.586 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 21:04:07 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-1-activate[75429]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-03-26T21:04:07.586 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 21:04:07 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-1-activate[75429]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-03-26T21:04:07.586 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 21:04:07 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-1-activate[75429]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-03-26T21:04:07.587 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 21:04:07 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-1-activate[75429]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-03-26T21:04:07.587 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 21:04:07 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-1-activate[75429]: --> ceph-volume raw activate successful for osd ID: 1 2024-03-26T21:04:07.587 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 21:04:07 smithi063 podman[75415]: 2024-03-26 21:04:07.508884068 +0000 UTC m=+1.495877948 container died 27d2f283b70a06aaa26570990b194c021f6faa1e33d1c34a51353dc998e7df59 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-1-activate, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, ceph=True, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-03-26T21:04:08.087 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:08 smithi063 ceph-mon[66934]: pgmap v105: 97 pgs: 3 activating+undersized, 22 active+undersized+degraded, 14 active+undersized, 58 active+clean; 339 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.2 MiB/s wr, 496 op/s; 176/1671 objects degraded (10.533%) 2024-03-26T21:04:08.509 INFO:teuthology.orchestra.run.smithi063.stdout: 5 216159 10.85 MB/sec execute 362 sec latency 43.215 ms 2024-03-26T21:04:08.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:08 smithi174 ceph-mon[53014]: pgmap v105: 97 pgs: 3 activating+undersized, 22 active+undersized+degraded, 14 active+undersized, 58 active+clean; 339 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.2 MiB/s wr, 496 op/s; 176/1671 objects degraded (10.533%) 2024-03-26T21:04:09.092 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 21:04:08 smithi063 podman[75555]: 2024-03-26 21:04:08.971466966 +0000 UTC m=+1.448572757 container remove 27d2f283b70a06aaa26570990b194c021f6faa1e33d1c34a51353dc998e7df59 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-1-activate, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240326) 2024-03-26T21:04:09.348 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:09 smithi063 ceph-mon[66934]: Health check update: Degraded data redundancy: 175/1659 objects degraded (10.549%), 23 pgs degraded (PG_DEGRADED) 2024-03-26T21:04:09.348 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 21:04:09 smithi063 podman[75606]: 2024-03-26 21:04:09.147575327 +0000 UTC m=+0.030746861 image pull a9bdc3eaeb106993bcbbac35d7fe5c7b76c052021676e129f5984aed30aee063 quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d 2024-03-26T21:04:09.348 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 21:04:09 smithi063 podman[75606]: 2024-03-26 21:04:09.28927384 +0000 UTC m=+0.172445371 container create d41fbe507197d662b5c28b38a0faee89b3db907f4774e9f34c8dc10b6b6a77e7 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-1, GIT_CLEAN=True, org.label-schema.build-date=20240326, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, org.label-schema.vendor=CentOS) 2024-03-26T21:04:09.509 INFO:teuthology.orchestra.run.smithi063.stdout: 5 216484 10.84 MB/sec execute 363 sec latency 32.784 ms 2024-03-26T21:04:09.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:09 smithi174 ceph-mon[53014]: Health check update: Degraded data redundancy: 175/1659 objects degraded (10.549%), 23 pgs degraded (PG_DEGRADED) 2024-03-26T21:04:09.599 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 21:04:09 smithi063 podman[75606]: 2024-03-26 21:04:09.592274125 +0000 UTC m=+0.475445664 container init d41fbe507197d662b5c28b38a0faee89b3db907f4774e9f34c8dc10b6b6a77e7 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-1, org.label-schema.build-date=20240326, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, GIT_BRANCH=HEAD, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5) 2024-03-26T21:04:09.599 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 21:04:09 smithi063 podman[75606]: 2024-03-26 21:04:09.601279826 +0000 UTC m=+0.484451366 container start d41fbe507197d662b5c28b38a0faee89b3db907f4774e9f34c8dc10b6b6a77e7 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-1, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , ceph=True, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, io.buildah.version=1.33.5) 2024-03-26T21:04:09.851 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 21:04:09 smithi063 ceph-osd[75624]: -- 172.21.15.63:0/1122515008 <== mon.0 v2:172.21.15.63:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5597987f3680 con 0x5597987bcc00 2024-03-26T21:04:09.851 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 21:04:09 smithi063 bash[75606]: d41fbe507197d662b5c28b38a0faee89b3db907f4774e9f34c8dc10b6b6a77e7 2024-03-26T21:04:09.851 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 21:04:09 smithi063 systemd[1]: Started Ceph osd.1 for 0ed4dd4c-ebb2-11ee-95d0-87774f69a715. 2024-03-26T21:04:10.360 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:10 smithi063 ceph-mon[66934]: pgmap v106: 97 pgs: 23 active+undersized+degraded, 16 active+undersized, 58 active+clean; 341 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.3 MiB/s wr, 638 op/s; 175/1659 objects degraded (10.549%) 2024-03-26T21:04:10.360 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:10 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:04:10.360 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:10 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:04:10.360 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:10 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:04:10.360 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 21:04:10 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-1[75620]: 2024-03-26T21:04:10.183+0000 7f39910827c0 -1 Falling back to public interface 2024-03-26T21:04:10.509 INFO:teuthology.orchestra.run.smithi063.stdout: 5 216833 10.84 MB/sec execute 364 sec latency 40.282 ms 2024-03-26T21:04:10.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:10 smithi174 ceph-mon[53014]: pgmap v106: 97 pgs: 23 active+undersized+degraded, 16 active+undersized, 58 active+clean; 341 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.3 MiB/s wr, 638 op/s; 175/1659 objects degraded (10.549%) 2024-03-26T21:04:10.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:10 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:04:10.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:10 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:04:10.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:10 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:04:11.510 INFO:teuthology.orchestra.run.smithi063.stdout: 5 217155 10.84 MB/sec execute 365 sec latency 76.580 ms 2024-03-26T21:04:11.920 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:11 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:04:11.921 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:11 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:04:11.921 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:11 smithi063 ceph-mon[66934]: pgmap v107: 97 pgs: 23 active+undersized+degraded, 16 active+undersized, 58 active+clean; 341 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.1 MiB/s wr, 514 op/s; 175/1659 objects degraded (10.549%) 2024-03-26T21:04:12.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:11 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:04:12.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:11 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:04:12.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:11 smithi174 ceph-mon[53014]: pgmap v107: 97 pgs: 23 active+undersized+degraded, 16 active+undersized, 58 active+clean; 341 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.1 MiB/s wr, 514 op/s; 175/1659 objects degraded (10.549%) 2024-03-26T21:04:12.509 INFO:teuthology.orchestra.run.smithi063.stdout: 5 217508 10.84 MB/sec execute 366 sec latency 60.583 ms 2024-03-26T21:04:13.509 INFO:teuthology.orchestra.run.smithi063.stdout: 5 217840 10.82 MB/sec execute 367 sec latency 43.030 ms 2024-03-26T21:04:14.205 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:13 smithi063 ceph-mon[66934]: pgmap v108: 97 pgs: 23 active+undersized+degraded, 16 active+undersized, 58 active+clean; 344 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.6 MiB/s wr, 523 op/s; 191/1650 objects degraded (11.576%) 2024-03-26T21:04:14.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:13 smithi174 ceph-mon[53014]: pgmap v108: 97 pgs: 23 active+undersized+degraded, 16 active+undersized, 58 active+clean; 344 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.6 MiB/s wr, 523 op/s; 191/1650 objects degraded (11.576%) 2024-03-26T21:04:14.510 INFO:teuthology.orchestra.run.smithi063.stdout: 5 218316 10.82 MB/sec execute 368 sec latency 38.402 ms 2024-03-26T21:04:15.209 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:14 smithi063 ceph-mon[66934]: Health check update: Degraded data redundancy: 191/1650 objects degraded (11.576%), 23 pgs degraded (PG_DEGRADED) 2024-03-26T21:04:15.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:14 smithi174 ceph-mon[53014]: Health check update: Degraded data redundancy: 191/1650 objects degraded (11.576%), 23 pgs degraded (PG_DEGRADED) 2024-03-26T21:04:15.510 INFO:teuthology.orchestra.run.smithi063.stdout: 5 218762 10.83 MB/sec execute 369 sec latency 35.966 ms 2024-03-26T21:04:16.223 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:15 smithi063 ceph-mon[66934]: pgmap v109: 97 pgs: 23 active+undersized+degraded, 16 active+undersized, 58 active+clean; 344 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.8 MiB/s wr, 464 op/s; 191/1650 objects degraded (11.576%) 2024-03-26T21:04:16.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:15 smithi174 ceph-mon[53014]: pgmap v109: 97 pgs: 23 active+undersized+degraded, 16 active+undersized, 58 active+clean; 344 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.8 MiB/s wr, 464 op/s; 191/1650 objects degraded (11.576%) 2024-03-26T21:04:16.509 INFO:teuthology.orchestra.run.smithi063.stdout: 5 219219 10.84 MB/sec execute 370 sec latency 46.131 ms 2024-03-26T21:04:17.479 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 21:04:17 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-1[75620]: 2024-03-26T21:04:17.457+0000 7f39910827c0 -1 osd.1 0 read_superblock omap replica is missing. 2024-03-26T21:04:17.510 INFO:teuthology.orchestra.run.smithi063.stdout: 5 219591 10.84 MB/sec execute 371 sec latency 53.578 ms 2024-03-26T21:04:18.243 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:18 smithi063 ceph-mon[66934]: pgmap v110: 97 pgs: 23 active+undersized+degraded, 16 active+undersized, 58 active+clean; 345 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 9.8 KiB/s rd, 6.8 MiB/s wr, 698 op/s; 174/1635 objects degraded (10.642%) 2024-03-26T21:04:18.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:18 smithi174 ceph-mon[53014]: pgmap v110: 97 pgs: 23 active+undersized+degraded, 16 active+undersized, 58 active+clean; 345 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 9.8 KiB/s rd, 6.8 MiB/s wr, 698 op/s; 174/1635 objects degraded (10.642%) 2024-03-26T21:04:18.510 INFO:teuthology.orchestra.run.smithi063.stdout: 5 219887 10.83 MB/sec execute 372 sec latency 40.199 ms 2024-03-26T21:04:19.015 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 21:04:18 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-1[75620]: 2024-03-26T21:04:18.773+0000 7f39910827c0 -1 osd.1 86 log_to_monitors true 2024-03-26T21:04:19.510 INFO:teuthology.orchestra.run.smithi063.stdout: 5 220210 10.82 MB/sec execute 373 sec latency 48.606 ms 2024-03-26T21:04:19.786 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 21:04:19 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-1[75620]: 2024-03-26T21:04:19.653+0000 7f39824da700 -1 osd.1 86 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-26T21:04:19.786 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:19 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:04:19.786 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:19 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:04:19.786 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:19 smithi063 ceph-mon[66934]: from='osd.1 [v2:172.21.15.63:6810/359826618,v1:172.21.15.63:6811/359826618]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-03-26T21:04:19.786 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:19 smithi063 ceph-mon[66934]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-03-26T21:04:19.786 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:19 smithi063 ceph-mon[66934]: pgmap v111: 97 pgs: 23 active+undersized+degraded, 16 active+undersized, 58 active+clean; 345 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.5 MiB/s wr, 561 op/s; 176/1662 objects degraded (10.590%) 2024-03-26T21:04:20.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:19 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:04:20.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:19 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:04:20.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:19 smithi174 ceph-mon[53014]: from='osd.1 [v2:172.21.15.63:6810/359826618,v1:172.21.15.63:6811/359826618]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-03-26T21:04:20.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:19 smithi174 ceph-mon[53014]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-03-26T21:04:20.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:19 smithi174 ceph-mon[53014]: pgmap v111: 97 pgs: 23 active+undersized+degraded, 16 active+undersized, 58 active+clean; 345 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.5 MiB/s wr, 561 op/s; 176/1662 objects degraded (10.590%) 2024-03-26T21:04:20.510 INFO:teuthology.orchestra.run.smithi063.stdout: 5 220581 10.83 MB/sec execute 374 sec latency 37.143 ms 2024-03-26T21:04:20.902 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:20 smithi063 ceph-mon[66934]: Health check update: Degraded data redundancy: 174/1635 objects degraded (10.642%), 23 pgs degraded (PG_DEGRADED) 2024-03-26T21:04:20.903 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:20 smithi063 ceph-mon[66934]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-03-26T21:04:20.903 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:20 smithi063 ceph-mon[66934]: from='osd.1 [v2:172.21.15.63:6810/359826618,v1:172.21.15.63:6811/359826618]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi063", "root=default"]}]: dispatch 2024-03-26T21:04:20.903 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:20 smithi063 ceph-mon[66934]: osdmap e89: 8 total, 7 up, 8 in 2024-03-26T21:04:20.903 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:20 smithi063 ceph-mon[66934]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi063", "root=default"]}]: dispatch 2024-03-26T21:04:20.903 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:20 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:04:20.903 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:20 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:04:21.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:20 smithi174 ceph-mon[53014]: Health check update: Degraded data redundancy: 174/1635 objects degraded (10.642%), 23 pgs degraded (PG_DEGRADED) 2024-03-26T21:04:21.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:20 smithi174 ceph-mon[53014]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-03-26T21:04:21.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:20 smithi174 ceph-mon[53014]: from='osd.1 [v2:172.21.15.63:6810/359826618,v1:172.21.15.63:6811/359826618]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi063", "root=default"]}]: dispatch 2024-03-26T21:04:21.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:20 smithi174 ceph-mon[53014]: osdmap e89: 8 total, 7 up, 8 in 2024-03-26T21:04:21.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:20 smithi174 ceph-mon[53014]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi063", "root=default"]}]: dispatch 2024-03-26T21:04:21.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:20 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:04:21.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:20 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:04:21.341 INFO:teuthology.orchestra.run.smithi063.stdout:true 2024-03-26T21:04:21.510 INFO:teuthology.orchestra.run.smithi063.stdout: 5 220615 10.80 MB/sec execute 375 sec latency 908.884 ms 2024-03-26T21:04:21.955 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:21 smithi063 ceph-mon[66934]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-26T21:04:21.955 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:21 smithi063 ceph-mon[66934]: osd.1 [v2:172.21.15.63:6810/359826618,v1:172.21.15.63:6811/359826618] boot 2024-03-26T21:04:21.955 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:21 smithi063 ceph-mon[66934]: osdmap e90: 8 total, 8 up, 8 in 2024-03-26T21:04:21.955 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:21 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-26T21:04:21.955 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:21 smithi063 ceph-mon[66934]: pgmap v114: 97 pgs: 23 active+undersized+degraded, 16 active+undersized, 58 active+clean; 345 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.6 MiB/s wr, 372 op/s; 176/1662 objects degraded (10.590%) 2024-03-26T21:04:22.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:21 smithi174 ceph-mon[53014]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-26T21:04:22.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:21 smithi174 ceph-mon[53014]: osd.1 [v2:172.21.15.63:6810/359826618,v1:172.21.15.63:6811/359826618] boot 2024-03-26T21:04:22.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:21 smithi174 ceph-mon[53014]: osdmap e90: 8 total, 8 up, 8 in 2024-03-26T21:04:22.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:21 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-26T21:04:22.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:21 smithi174 ceph-mon[53014]: pgmap v114: 97 pgs: 23 active+undersized+degraded, 16 active+undersized, 58 active+clean; 345 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.6 MiB/s wr, 372 op/s; 176/1662 objects degraded (10.590%) 2024-03-26T21:04:22.073 INFO:teuthology.orchestra.run.smithi063.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-26T21:04:22.073 INFO:teuthology.orchestra.run.smithi063.stdout:alertmanager.smithi063 smithi063 *:9093,9094 running (5m) 3s ago 14m 24.5M - 0.23.0 ba2b418f427c 6e160c94aac3 2024-03-26T21:04:22.073 INFO:teuthology.orchestra.run.smithi063.stdout:crash.smithi063 smithi063 running (100s) 3s ago 14m 6665k - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 cf1096d84270 2024-03-26T21:04:22.073 INFO:teuthology.orchestra.run.smithi063.stdout:crash.smithi174 smithi174 running (94s) 89s ago 14m 6665k - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 cb5371a92684 2024-03-26T21:04:22.073 INFO:teuthology.orchestra.run.smithi063.stdout:grafana.smithi063 smithi063 *:3000 error 3s ago 14m - - 2024-03-26T21:04:22.073 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foofs.smithi063.tzaznz smithi063 running (8m) 3s ago 8m 37.9M - 17.2.0 e1d6a67b021e 10a989fc7954 2024-03-26T21:04:22.073 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foofs.smithi174.gitdvk smithi174 running (8m) 89s ago 8m 13.4M - 17.2.0 e1d6a67b021e 4339a5d98d1d 2024-03-26T21:04:22.074 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.smithi063.lxhujt smithi063 *:8443,9283,8765 running (4m) 3s ago 15m 505M - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 6d9d10330a90 2024-03-26T21:04:22.074 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.smithi174.sexoin smithi174 *:8443,9283,8765 running (3m) 89s ago 14m 450M - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 d7546233672d 2024-03-26T21:04:22.074 INFO:teuthology.orchestra.run.smithi063.stdout:mon.smithi063 smithi063 running (3m) 3s ago 16m 50.2M 2048M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 c644200344f8 2024-03-26T21:04:22.074 INFO:teuthology.orchestra.run.smithi063.stdout:mon.smithi174 smithi174 running (2m) 89s ago 13m 39.0M 2048M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 1a6d442692e7 2024-03-26T21:04:22.074 INFO:teuthology.orchestra.run.smithi063.stdout:nfs.foo.0.0.smithi063.fgmpwa smithi063 *:2049 running (8m) 3s ago 8m 277M - 4.0 e1d6a67b021e 3d6c0dd9ce51 2024-03-26T21:04:22.074 INFO:teuthology.orchestra.run.smithi063.stdout:nfs.foo.1.0.smithi174.pytptt smithi174 *:2049 running (8m) 89s ago 8m 86.4M - 4.0 e1d6a67b021e f220dbb449a3 2024-03-26T21:04:22.074 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.smithi063 smithi063 *:9100 running (5m) 3s ago 14m 13.9M - 1.3.1 1dbe0e931976 f52f41fd14c0 2024-03-26T21:04:22.074 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.smithi174 smithi174 *:9100 running (4m) 89s ago 13m 17.2M - 1.3.1 1dbe0e931976 0c027f39a4c8 2024-03-26T21:04:22.074 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0 smithi063 running (61s) 3s ago 13m 259M 1936M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 2d4b3fa825cf 2024-03-26T21:04:22.074 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1 smithi063 running (12s) 3s ago 12m 15.0M 1936M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 d41fbe507197 2024-03-26T21:04:22.074 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2 smithi063 running (12m) 3s ago 12m 499M 1936M 17.2.0 e1d6a67b021e 107f685ad08d 2024-03-26T21:04:22.074 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3 smithi063 running (11m) 3s ago 11m 442M 1936M 17.2.0 e1d6a67b021e ab0f1a855957 2024-03-26T21:04:22.074 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4 smithi174 running (11m) 89s ago 11m 575M 2704M 17.2.0 e1d6a67b021e a0e877ecc06d 2024-03-26T21:04:22.074 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5 smithi174 running (11m) 89s ago 11m 423M 2704M 17.2.0 e1d6a67b021e 378c2aca7e71 2024-03-26T21:04:22.075 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6 smithi174 running (10m) 89s ago 10m 328M 2704M 17.2.0 e1d6a67b021e ffbb5b7ad3b6 2024-03-26T21:04:22.075 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7 smithi174 running (10m) 89s ago 10m 524M 2704M 17.2.0 e1d6a67b021e c1d57fcabb3f 2024-03-26T21:04:22.075 INFO:teuthology.orchestra.run.smithi063.stdout:prometheus.smithi063 smithi063 *:9095 running (3m) 3s ago 13m 67.4M - 2.33.4 514e6a882f6e 567eef202627 2024-03-26T21:04:22.510 INFO:teuthology.orchestra.run.smithi063.stdout: 5 220615 10.77 MB/sec execute 376 sec latency 1908.911 ms 2024-03-26T21:04:22.518 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-03-26T21:04:22.519 INFO:teuthology.orchestra.run.smithi063.stdout: "mon": { 2024-03-26T21:04:22.519 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 2 2024-03-26T21:04:22.519 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:04:22.519 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr": { 2024-03-26T21:04:22.519 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 2 2024-03-26T21:04:22.519 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:04:22.519 INFO:teuthology.orchestra.run.smithi063.stdout: "osd": { 2024-03-26T21:04:22.519 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-03-26T21:04:22.519 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 2 2024-03-26T21:04:22.519 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:04:22.519 INFO:teuthology.orchestra.run.smithi063.stdout: "mds": { 2024-03-26T21:04:22.519 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-26T21:04:22.519 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:04:22.520 INFO:teuthology.orchestra.run.smithi063.stdout: "overall": { 2024-03-26T21:04:22.520 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8, 2024-03-26T21:04:22.520 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 6 2024-03-26T21:04:22.520 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-03-26T21:04:22.520 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-03-26T21:04:22.900 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-03-26T21:04:22.900 INFO:teuthology.orchestra.run.smithi063.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d", 2024-03-26T21:04:22.900 INFO:teuthology.orchestra.run.smithi063.stdout: "in_progress": true, 2024-03-26T21:04:22.901 INFO:teuthology.orchestra.run.smithi063.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-26T21:04:22.901 INFO:teuthology.orchestra.run.smithi063.stdout: "services_complete": [ 2024-03-26T21:04:22.901 INFO:teuthology.orchestra.run.smithi063.stdout: "crash", 2024-03-26T21:04:22.901 INFO:teuthology.orchestra.run.smithi063.stdout: "mon", 2024-03-26T21:04:22.901 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr" 2024-03-26T21:04:22.901 INFO:teuthology.orchestra.run.smithi063.stdout: ], 2024-03-26T21:04:22.901 INFO:teuthology.orchestra.run.smithi063.stdout: "progress": "8/23 daemons upgraded", 2024-03-26T21:04:22.901 INFO:teuthology.orchestra.run.smithi063.stdout: "message": "Currently upgrading osd daemons", 2024-03-26T21:04:22.901 INFO:teuthology.orchestra.run.smithi063.stdout: "is_paused": false 2024-03-26T21:04:22.901 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-03-26T21:04:22.901 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:22 smithi063 ceph-mon[66934]: from='client.34256 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:04:22.901 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:22 smithi063 ceph-mon[66934]: osdmap e91: 8 total, 8 up, 8 in 2024-03-26T21:04:22.902 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:22 smithi063 ceph-mon[66934]: from='client.34260 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:04:22.902 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:22 smithi063 ceph-mon[66934]: from='client.34264 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:04:22.902 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:22 smithi063 ceph-mon[66934]: from='client.? 172.21.15.63:0/523320757' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:04:23.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:22 smithi174 ceph-mon[53014]: from='client.34256 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:04:23.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:22 smithi174 ceph-mon[53014]: osdmap e91: 8 total, 8 up, 8 in 2024-03-26T21:04:23.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:22 smithi174 ceph-mon[53014]: from='client.34260 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:04:23.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:22 smithi174 ceph-mon[53014]: from='client.34264 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:04:23.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:22 smithi174 ceph-mon[53014]: from='client.? 172.21.15.63:0/523320757' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:04:23.510 INFO:teuthology.orchestra.run.smithi063.stdout: 5 220713 10.75 MB/sec execute 377 sec latency 2421.798 ms 2024-03-26T21:04:23.582 INFO:teuthology.orchestra.run.smithi063.stdout:HEALTH_WARN 1 failed cephadm daemon(s); Degraded data redundancy: 176/1662 objects degraded (10.590%), 23 pgs degraded 2024-03-26T21:04:23.582 INFO:teuthology.orchestra.run.smithi063.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-26T21:04:23.583 INFO:teuthology.orchestra.run.smithi063.stdout: daemon grafana.smithi063 on smithi063 is in error state 2024-03-26T21:04:23.583 INFO:teuthology.orchestra.run.smithi063.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 176/1662 objects degraded (10.590%), 23 pgs degraded 2024-03-26T21:04:23.583 INFO:teuthology.orchestra.run.smithi063.stdout: pg 2.0 is active+undersized+degraded, acting [7,0] 2024-03-26T21:04:23.583 INFO:teuthology.orchestra.run.smithi063.stdout: pg 2.2 is active+undersized+degraded, acting [5,6] 2024-03-26T21:04:23.583 INFO:teuthology.orchestra.run.smithi063.stdout: pg 2.4 is active+undersized+degraded, acting [0,7] 2024-03-26T21:04:23.583 INFO:teuthology.orchestra.run.smithi063.stdout: pg 2.6 is active+undersized+degraded, acting [6,4] 2024-03-26T21:04:23.583 INFO:teuthology.orchestra.run.smithi063.stdout: pg 2.8 is active+undersized+degraded, acting [7,0] 2024-03-26T21:04:23.583 INFO:teuthology.orchestra.run.smithi063.stdout: pg 2.9 is active+undersized+degraded, acting [7,3] 2024-03-26T21:04:23.583 INFO:teuthology.orchestra.run.smithi063.stdout: pg 2.a is active+undersized+degraded, acting [3,7] 2024-03-26T21:04:23.583 INFO:teuthology.orchestra.run.smithi063.stdout: pg 2.d is active+undersized+degraded, acting [4,3] 2024-03-26T21:04:23.583 INFO:teuthology.orchestra.run.smithi063.stdout: pg 2.10 is active+undersized+degraded, acting [2,0] 2024-03-26T21:04:23.583 INFO:teuthology.orchestra.run.smithi063.stdout: pg 2.11 is active+undersized+degraded, acting [6,4] 2024-03-26T21:04:23.583 INFO:teuthology.orchestra.run.smithi063.stdout: pg 2.15 is active+undersized+degraded, acting [5,0] 2024-03-26T21:04:23.584 INFO:teuthology.orchestra.run.smithi063.stdout: pg 3.0 is active+undersized+degraded, acting [2,6] 2024-03-26T21:04:23.584 INFO:teuthology.orchestra.run.smithi063.stdout: pg 3.4 is active+undersized+degraded, acting [2,5] 2024-03-26T21:04:23.584 INFO:teuthology.orchestra.run.smithi063.stdout: pg 3.6 is active+undersized+degraded, acting [0,4] 2024-03-26T21:04:23.584 INFO:teuthology.orchestra.run.smithi063.stdout: pg 3.8 is active+undersized+degraded, acting [3,7] 2024-03-26T21:04:23.584 INFO:teuthology.orchestra.run.smithi063.stdout: pg 3.a is active+undersized+degraded, acting [6,4] 2024-03-26T21:04:23.584 INFO:teuthology.orchestra.run.smithi063.stdout: pg 3.e is active+undersized+degraded, acting [7,4] 2024-03-26T21:04:23.584 INFO:teuthology.orchestra.run.smithi063.stdout: pg 3.16 is active+undersized+degraded, acting [5,7] 2024-03-26T21:04:23.584 INFO:teuthology.orchestra.run.smithi063.stdout: pg 3.18 is active+undersized+degraded, acting [3,0] 2024-03-26T21:04:23.584 INFO:teuthology.orchestra.run.smithi063.stdout: pg 3.19 is active+undersized+degraded, acting [3,4] 2024-03-26T21:04:23.584 INFO:teuthology.orchestra.run.smithi063.stdout: pg 3.1a is active+undersized+degraded, acting [4,2] 2024-03-26T21:04:23.584 INFO:teuthology.orchestra.run.smithi063.stdout: pg 3.1c is active+undersized+degraded, acting [5,4] 2024-03-26T21:04:23.584 INFO:teuthology.orchestra.run.smithi063.stdout: pg 4.a is active+undersized+degraded, acting [6,7] 2024-03-26T21:04:23.676 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:23 smithi063 ceph-mon[66934]: osdmap e92: 8 total, 8 up, 8 in 2024-03-26T21:04:23.676 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:23 smithi063 ceph-mon[66934]: from='client.34272 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:04:23.676 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:23 smithi063 ceph-mon[66934]: pgmap v117: 97 pgs: 4 activating+degraded, 11 remapped+peering, 5 activating, 5 active+undersized+degraded, 5 active+undersized, 67 active+clean; 348 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 5.1 MiB/s wr, 498 op/s; 56/1611 objects degraded (3.476%) 2024-03-26T21:04:23.676 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:23 smithi063 ceph-mon[66934]: from='client.? 172.21.15.63:0/1626532297' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-26T21:04:24.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:23 smithi174 ceph-mon[53014]: osdmap e92: 8 total, 8 up, 8 in 2024-03-26T21:04:24.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:23 smithi174 ceph-mon[53014]: from='client.34272 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:04:24.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:23 smithi174 ceph-mon[53014]: pgmap v117: 97 pgs: 4 activating+degraded, 11 remapped+peering, 5 activating, 5 active+undersized+degraded, 5 active+undersized, 67 active+clean; 348 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 5.1 MiB/s wr, 498 op/s; 56/1611 objects degraded (3.476%) 2024-03-26T21:04:24.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:23 smithi174 ceph-mon[53014]: from='client.? 172.21.15.63:0/1626532297' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-26T21:04:24.511 INFO:teuthology.orchestra.run.smithi063.stdout: 5 221057 10.74 MB/sec execute 378 sec latency 34.709 ms 2024-03-26T21:04:24.717 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:24 smithi063 ceph-mon[66934]: osdmap e93: 8 total, 8 up, 8 in 2024-03-26T21:04:24.717 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:24 smithi063 ceph-mon[66934]: Health check update: Degraded data redundancy: 56/1611 objects degraded (3.476%), 9 pgs degraded (PG_DEGRADED) 2024-03-26T21:04:24.717 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:24 smithi063 ceph-mon[66934]: osdmap e94: 8 total, 8 up, 8 in 2024-03-26T21:04:25.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:24 smithi174 ceph-mon[53014]: osdmap e93: 8 total, 8 up, 8 in 2024-03-26T21:04:25.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:24 smithi174 ceph-mon[53014]: Health check update: Degraded data redundancy: 56/1611 objects degraded (3.476%), 9 pgs degraded (PG_DEGRADED) 2024-03-26T21:04:25.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:24 smithi174 ceph-mon[53014]: osdmap e94: 8 total, 8 up, 8 in 2024-03-26T21:04:25.510 INFO:teuthology.orchestra.run.smithi063.stdout: 5 221111 10.72 MB/sec execute 379 sec latency 926.413 ms 2024-03-26T21:04:26.057 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:25 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:04:26.057 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:25 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:04:26.057 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:25 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:04:26.057 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:25 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T21:04:26.057 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:25 smithi063 ceph-mon[66934]: pgmap v120: 97 pgs: 4 activating+degraded, 10 remapped+peering, 5 activating, 78 active+clean; 348 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 5.2 MiB/s wr, 516 op/s; 35/1611 objects degraded (2.173%); 1.0 MiB/s, 61 keys/s, 0 objects/s recovering 2024-03-26T21:04:26.057 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:25 smithi063 ceph-mon[66934]: pgmap v121: 97 pgs: 4 activating+degraded, 10 remapped+peering, 5 activating, 78 active+clean; 348 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 14 KiB/s rd, 6.3 MiB/s wr, 624 op/s; 35/1611 objects degraded (2.173%); 1.2 MiB/s, 74 keys/s, 0 objects/s recovering 2024-03-26T21:04:26.057 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:25 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:04:26.057 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:25 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:04:26.057 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:25 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:04:26.058 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:25 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:04:26.058 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:25 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:04:26.058 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:25 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:04:26.058 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:25 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-26T21:04:26.058 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:25 smithi063 ceph-mon[66934]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-26T21:04:26.058 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:25 smithi063 ceph-mon[66934]: Upgrade: unsafe to stop osd(s) at this time (4 PGs are or would become offline) 2024-03-26T21:04:26.058 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:25 smithi063 ceph-mon[66934]: osdmap e95: 8 total, 8 up, 8 in 2024-03-26T21:04:26.058 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:25 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:04:26.058 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:25 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:04:26.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:25 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:04:26.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:25 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:04:26.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:25 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:04:26.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:25 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T21:04:26.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:25 smithi174 ceph-mon[53014]: pgmap v120: 97 pgs: 4 activating+degraded, 10 remapped+peering, 5 activating, 78 active+clean; 348 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 5.2 MiB/s wr, 516 op/s; 35/1611 objects degraded (2.173%); 1.0 MiB/s, 61 keys/s, 0 objects/s recovering 2024-03-26T21:04:26.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:25 smithi174 ceph-mon[53014]: pgmap v121: 97 pgs: 4 activating+degraded, 10 remapped+peering, 5 activating, 78 active+clean; 348 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 14 KiB/s rd, 6.3 MiB/s wr, 624 op/s; 35/1611 objects degraded (2.173%); 1.2 MiB/s, 74 keys/s, 0 objects/s recovering 2024-03-26T21:04:26.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:25 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:04:26.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:25 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:04:26.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:25 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:04:26.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:25 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:04:26.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:25 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:04:26.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:25 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:04:26.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:25 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-26T21:04:26.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:25 smithi174 ceph-mon[53014]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-26T21:04:26.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:25 smithi174 ceph-mon[53014]: Upgrade: unsafe to stop osd(s) at this time (4 PGs are or would become offline) 2024-03-26T21:04:26.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:25 smithi174 ceph-mon[53014]: osdmap e95: 8 total, 8 up, 8 in 2024-03-26T21:04:26.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:25 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:04:26.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:25 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:04:26.510 INFO:teuthology.orchestra.run.smithi063.stdout: 5 221281 10.70 MB/sec execute 380 sec latency 1048.827 ms 2024-03-26T21:04:27.510 INFO:teuthology.orchestra.run.smithi063.stdout: 5 221670 10.69 MB/sec execute 381 sec latency 817.913 ms 2024-03-26T21:04:27.842 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:27 smithi063 ceph-mon[66934]: osdmap e96: 8 total, 8 up, 8 in 2024-03-26T21:04:27.842 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:27 smithi063 ceph-mon[66934]: pgmap v124: 97 pgs: 4 remapped+peering, 2 peering, 91 active+clean; 345 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 4.1 KiB/s rd, 2.6 MiB/s wr, 319 op/s; 2.2 MiB/s, 74 keys/s, 56 objects/s recovering 2024-03-26T21:04:28.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:27 smithi174 ceph-mon[53014]: osdmap e96: 8 total, 8 up, 8 in 2024-03-26T21:04:28.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:27 smithi174 ceph-mon[53014]: pgmap v124: 97 pgs: 4 remapped+peering, 2 peering, 91 active+clean; 345 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 4.1 KiB/s rd, 2.6 MiB/s wr, 319 op/s; 2.2 MiB/s, 74 keys/s, 56 objects/s recovering 2024-03-26T21:04:28.510 INFO:teuthology.orchestra.run.smithi063.stdout: 5 222027 10.69 MB/sec execute 382 sec latency 744.030 ms 2024-03-26T21:04:28.847 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:28 smithi063 ceph-mon[66934]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 35/1611 objects degraded (2.173%), 4 pgs degraded) 2024-03-26T21:04:28.847 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:28 smithi063 ceph-mon[66934]: osdmap e97: 8 total, 8 up, 8 in 2024-03-26T21:04:29.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:28 smithi174 ceph-mon[53014]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 35/1611 objects degraded (2.173%), 4 pgs degraded) 2024-03-26T21:04:29.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:28 smithi174 ceph-mon[53014]: osdmap e97: 8 total, 8 up, 8 in 2024-03-26T21:04:29.510 INFO:teuthology.orchestra.run.smithi063.stdout: 5 222440 10.69 MB/sec execute 383 sec latency 53.438 ms 2024-03-26T21:04:29.602 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:29 smithi063 ceph-mon[66934]: pgmap v126: 97 pgs: 3 remapped+peering, 2 peering, 92 active+clean; 345 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 3.2 KiB/s rd, 2.6 MiB/s wr, 278 op/s; 921 KiB/s, 49 objects/s recovering 2024-03-26T21:04:30.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:29 smithi174 ceph-mon[53014]: pgmap v126: 97 pgs: 3 remapped+peering, 2 peering, 92 active+clean; 345 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 3.2 KiB/s rd, 2.6 MiB/s wr, 278 op/s; 921 KiB/s, 49 objects/s recovering 2024-03-26T21:04:30.511 INFO:teuthology.orchestra.run.smithi063.stdout: 5 222711 10.68 MB/sec execute 384 sec latency 62.723 ms 2024-03-26T21:04:31.510 INFO:teuthology.orchestra.run.smithi063.stdout: 5 222961 10.67 MB/sec execute 385 sec latency 48.229 ms 2024-03-26T21:04:31.904 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:31 smithi063 ceph-mon[66934]: pgmap v127: 97 pgs: 2 peering, 95 active+clean; 344 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 2.2 KiB/s rd, 1.9 MiB/s wr, 199 op/s; 614 KiB/s, 45 objects/s recovering 2024-03-26T21:04:32.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:31 smithi174 ceph-mon[53014]: pgmap v127: 97 pgs: 2 peering, 95 active+clean; 344 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 2.2 KiB/s rd, 1.9 MiB/s wr, 199 op/s; 614 KiB/s, 45 objects/s recovering 2024-03-26T21:04:32.510 INFO:teuthology.orchestra.run.smithi063.stdout: 5 223202 10.66 MB/sec execute 386 sec latency 63.777 ms 2024-03-26T21:04:33.510 INFO:teuthology.orchestra.run.smithi063.stdout: 5 223484 10.67 MB/sec execute 387 sec latency 42.725 ms 2024-03-26T21:04:34.010 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:33 smithi063 ceph-mon[66934]: pgmap v128: 97 pgs: 97 active+clean; 346 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.7 MiB/s wr, 483 op/s; 510 KiB/s, 37 objects/s recovering 2024-03-26T21:04:34.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:33 smithi174 ceph-mon[53014]: pgmap v128: 97 pgs: 97 active+clean; 346 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.7 MiB/s wr, 483 op/s; 510 KiB/s, 37 objects/s recovering 2024-03-26T21:04:34.510 INFO:teuthology.orchestra.run.smithi063.stdout: 5 223806 10.67 MB/sec execute 388 sec latency 68.967 ms 2024-03-26T21:04:35.512 INFO:teuthology.orchestra.run.smithi063.stdout: 5 224222 10.67 MB/sec execute 389 sec latency 33.307 ms 2024-03-26T21:04:35.923 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:35 smithi063 ceph-mon[66934]: pgmap v129: 97 pgs: 97 active+clean; 348 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.5 MiB/s wr, 465 op/s; 448 KiB/s, 32 objects/s recovering 2024-03-26T21:04:36.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:35 smithi174 ceph-mon[53014]: pgmap v129: 97 pgs: 97 active+clean; 348 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.5 MiB/s wr, 465 op/s; 448 KiB/s, 32 objects/s recovering 2024-03-26T21:04:36.512 INFO:teuthology.orchestra.run.smithi063.stdout: 5 224614 10.65 MB/sec execute 390 sec latency 52.018 ms 2024-03-26T21:04:37.512 INFO:teuthology.orchestra.run.smithi063.stdout: 5 225058 10.65 MB/sec execute 391 sec latency 43.060 ms 2024-03-26T21:04:37.942 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:37 smithi063 ceph-mon[66934]: pgmap v130: 97 pgs: 97 active+clean; 350 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.3 MiB/s wr, 401 op/s; 60 KiB/s, 9 objects/s recovering 2024-03-26T21:04:38.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:37 smithi174 ceph-mon[53014]: pgmap v130: 97 pgs: 97 active+clean; 350 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.3 MiB/s wr, 401 op/s; 60 KiB/s, 9 objects/s recovering 2024-03-26T21:04:38.512 INFO:teuthology.orchestra.run.smithi063.stdout: 5 225579 10.66 MB/sec execute 392 sec latency 44.890 ms 2024-03-26T21:04:39.512 INFO:teuthology.orchestra.run.smithi063.stdout: 5 226052 10.67 MB/sec execute 393 sec latency 33.304 ms 2024-03-26T21:04:39.962 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:39 smithi063 ceph-mon[66934]: pgmap v131: 97 pgs: 97 active+clean; 349 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.4 KiB/s rd, 4.1 MiB/s wr, 369 op/s; 0 B/s, 6 objects/s recovering 2024-03-26T21:04:40.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:39 smithi174 ceph-mon[53014]: pgmap v131: 97 pgs: 97 active+clean; 349 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.4 KiB/s rd, 4.1 MiB/s wr, 369 op/s; 0 B/s, 6 objects/s recovering 2024-03-26T21:04:40.514 INFO:teuthology.orchestra.run.smithi063.stdout: 5 226480 10.67 MB/sec execute 394 sec latency 42.226 ms 2024-03-26T21:04:40.978 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:40 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-26T21:04:40.979 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:40 smithi063 ceph-mon[66934]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-26T21:04:40.979 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:40 smithi063 ceph-mon[66934]: Upgrade: osd.2 is safe to restart 2024-03-26T21:04:40.979 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:40 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:04:40.979 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:40 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:04:40.979 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:40 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:04:41.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:40 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-26T21:04:41.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:40 smithi174 ceph-mon[53014]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-26T21:04:41.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:40 smithi174 ceph-mon[53014]: Upgrade: osd.2 is safe to restart 2024-03-26T21:04:41.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:40 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:04:41.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:40 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:04:41.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:40 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:04:41.514 INFO:teuthology.orchestra.run.smithi063.stdout: 5 226822 10.67 MB/sec execute 395 sec latency 38.472 ms 2024-03-26T21:04:41.993 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:41 smithi063 ceph-mon[66934]: pgmap v132: 97 pgs: 97 active+clean; 352 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.1 KiB/s rd, 4.4 MiB/s wr, 440 op/s; 0 B/s, 5 objects/s recovering 2024-03-26T21:04:42.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:41 smithi174 ceph-mon[53014]: pgmap v132: 97 pgs: 97 active+clean; 352 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.1 KiB/s rd, 4.4 MiB/s wr, 440 op/s; 0 B/s, 5 objects/s recovering 2024-03-26T21:04:42.514 INFO:teuthology.orchestra.run.smithi063.stdout: 5 227114 10.67 MB/sec execute 396 sec latency 63.769 ms 2024-03-26T21:04:43.514 INFO:teuthology.orchestra.run.smithi063.stdout: 5 227345 10.66 MB/sec execute 397 sec latency 69.362 ms 2024-03-26T21:04:44.047 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:44 smithi063 ceph-mon[66934]: pgmap v133: 97 pgs: 97 active+clean; 344 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 9.0 KiB/s rd, 6.5 MiB/s wr, 607 op/s 2024-03-26T21:04:44.047 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:44 smithi063 ceph-mon[66934]: Upgrade: Updating osd.2 2024-03-26T21:04:44.047 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:44 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:04:44.047 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:44 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-03-26T21:04:44.047 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:44 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:04:44.047 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:44 smithi063 ceph-mon[66934]: Deploying daemon osd.2 on smithi063 2024-03-26T21:04:44.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:44 smithi174 ceph-mon[53014]: pgmap v133: 97 pgs: 97 active+clean; 344 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 9.0 KiB/s rd, 6.5 MiB/s wr, 607 op/s 2024-03-26T21:04:44.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:44 smithi174 ceph-mon[53014]: Upgrade: Updating osd.2 2024-03-26T21:04:44.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:44 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:04:44.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:44 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-03-26T21:04:44.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:44 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:04:44.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:44 smithi174 ceph-mon[53014]: Deploying daemon osd.2 on smithi063 2024-03-26T21:04:44.514 INFO:teuthology.orchestra.run.smithi063.stdout: 5 227599 10.66 MB/sec execute 398 sec latency 55.369 ms 2024-03-26T21:04:45.514 INFO:teuthology.orchestra.run.smithi063.stdout: 5 227758 10.64 MB/sec execute 399 sec latency 142.149 ms 2024-03-26T21:04:46.104 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:46 smithi063 ceph-mon[66934]: pgmap v134: 97 pgs: 97 active+clean; 347 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.5 KiB/s rd, 5.0 MiB/s wr, 485 op/s 2024-03-26T21:04:46.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:46 smithi174 ceph-mon[53014]: pgmap v134: 97 pgs: 97 active+clean; 347 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.5 KiB/s rd, 5.0 MiB/s wr, 485 op/s 2024-03-26T21:04:46.514 INFO:teuthology.orchestra.run.smithi063.stdout: 5 228009 10.62 MB/sec execute 400 sec latency 58.381 ms 2024-03-26T21:04:47.514 INFO:teuthology.orchestra.run.smithi063.stdout: 5 228288 10.61 MB/sec execute 401 sec latency 52.909 ms 2024-03-26T21:04:48.125 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:48 smithi063 ceph-mon[66934]: pgmap v135: 97 pgs: 97 active+clean; 348 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 9.4 KiB/s rd, 5.9 MiB/s wr, 587 op/s 2024-03-26T21:04:48.125 INFO:journalctl@ceph.osd.2.smithi063.stdout:Mar 26 21:04:48 smithi063 systemd[1]: Stopping Ceph osd.2 for 0ed4dd4c-ebb2-11ee-95d0-87774f69a715... 2024-03-26T21:04:48.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:48 smithi174 ceph-mon[53014]: pgmap v135: 97 pgs: 97 active+clean; 348 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 9.4 KiB/s rd, 5.9 MiB/s wr, 587 op/s 2024-03-26T21:04:48.514 INFO:teuthology.orchestra.run.smithi063.stdout: 5 228579 10.59 MB/sec execute 402 sec latency 51.855 ms 2024-03-26T21:04:48.657 INFO:journalctl@ceph.osd.2.smithi063.stdout:Mar 26 21:04:48 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-2[43070]: 2024-03-26T21:04:48.465+0000 7f63fac29700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-26T21:04:48.657 INFO:journalctl@ceph.osd.2.smithi063.stdout:Mar 26 21:04:48 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-2[43070]: 2024-03-26T21:04:48.465+0000 7f63fac29700 -1 osd.2 97 *** Got signal Terminated *** 2024-03-26T21:04:48.657 INFO:journalctl@ceph.osd.2.smithi063.stdout:Mar 26 21:04:48 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-2[43070]: 2024-03-26T21:04:48.465+0000 7f63fac29700 -1 osd.2 97 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-26T21:04:49.161 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:49 smithi063 ceph-mon[66934]: osd.2 marked itself down and dead 2024-03-26T21:04:49.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:49 smithi174 ceph-mon[53014]: osd.2 marked itself down and dead 2024-03-26T21:04:49.514 INFO:teuthology.orchestra.run.smithi063.stdout: 5 228862 10.59 MB/sec execute 403 sec latency 463.271 ms 2024-03-26T21:04:49.969 INFO:journalctl@ceph.osd.2.smithi063.stdout:Mar 26 21:04:49 smithi063 podman[77710]: 2024-03-26 21:04:49.696873798 +0000 UTC m=+1.369978570 container died 107f685ad08d6fe550e79837efda83d388b25103fa7a2653e2d6dc0a1c660af4 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-2, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, distribution-scope=public, com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, release=754, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-03-26T21:04:50.231 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:50 smithi063 ceph-mon[66934]: pgmap v136: 97 pgs: 97 active+clean; 347 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.9 MiB/s wr, 498 op/s 2024-03-26T21:04:50.231 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:50 smithi063 ceph-mon[66934]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-26T21:04:50.231 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:50 smithi063 ceph-mon[66934]: osdmap e98: 8 total, 7 up, 8 in 2024-03-26T21:04:50.231 INFO:journalctl@ceph.osd.2.smithi063.stdout:Mar 26 21:04:50 smithi063 podman[77710]: 2024-03-26 21:04:50.175155253 +0000 UTC m=+1.848260023 container cleanup 107f685ad08d6fe550e79837efda83d388b25103fa7a2653e2d6dc0a1c660af4 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-2, ceph=True, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, architecture=x86_64, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, version=8) 2024-03-26T21:04:50.231 INFO:journalctl@ceph.osd.2.smithi063.stdout:Mar 26 21:04:50 smithi063 bash[77710]: ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-2 2024-03-26T21:04:50.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:50 smithi174 ceph-mon[53014]: pgmap v136: 97 pgs: 97 active+clean; 347 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.9 MiB/s wr, 498 op/s 2024-03-26T21:04:50.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:50 smithi174 ceph-mon[53014]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-26T21:04:50.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:50 smithi174 ceph-mon[53014]: osdmap e98: 8 total, 7 up, 8 in 2024-03-26T21:04:50.514 INFO:teuthology.orchestra.run.smithi063.stdout: 5 228968 10.57 MB/sec execute 404 sec latency 1042.677 ms 2024-03-26T21:04:50.987 INFO:journalctl@ceph.osd.2.smithi063.stdout:Mar 26 21:04:50 smithi063 podman[77727]: 2024-03-26 21:04:50.752749621 +0000 UTC m=+1.054971435 container remove 107f685ad08d6fe550e79837efda83d388b25103fa7a2653e2d6dc0a1c660af4 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-2, vcs-type=git, ceph=True, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, RELEASE=HEAD, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container) 2024-03-26T21:04:51.245 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:51 smithi063 ceph-mon[66934]: osdmap e99: 8 total, 7 up, 8 in 2024-03-26T21:04:51.245 INFO:journalctl@ceph.osd.2.smithi063.stdout:Mar 26 21:04:51 smithi063 podman[77807]: 2024-03-26 21:04:51.10876468 +0000 UTC m=+0.026545529 image pull a9bdc3eaeb106993bcbbac35d7fe5c7b76c052021676e129f5984aed30aee063 quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d 2024-03-26T21:04:51.245 INFO:journalctl@ceph.osd.2.smithi063.stdout:Mar 26 21:04:51 smithi063 podman[77807]: 2024-03-26 21:04:51.240337208 +0000 UTC m=+0.158118080 container create c2284be5ab1a0c30830f3c20b794eb5870ed604a3d99b8592bf3bc7953a92a0c (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-2-deactivate, io.buildah.version=1.33.5, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-03-26T21:04:51.514 INFO:teuthology.orchestra.run.smithi063.stdout: 5 229311 10.56 MB/sec execute 405 sec latency 38.258 ms 2024-03-26T21:04:51.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:51 smithi174 ceph-mon[53014]: osdmap e99: 8 total, 7 up, 8 in 2024-03-26T21:04:51.754 INFO:journalctl@ceph.osd.2.smithi063.stdout:Mar 26 21:04:51 smithi063 podman[77807]: 2024-03-26 21:04:51.531564807 +0000 UTC m=+0.449345640 container init c2284be5ab1a0c30830f3c20b794eb5870ed604a3d99b8592bf3bc7953a92a0c (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-2-deactivate, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=) 2024-03-26T21:04:51.754 INFO:journalctl@ceph.osd.2.smithi063.stdout:Mar 26 21:04:51 smithi063 podman[77807]: 2024-03-26 21:04:51.540435302 +0000 UTC m=+0.458216136 container start c2284be5ab1a0c30830f3c20b794eb5870ed604a3d99b8592bf3bc7953a92a0c (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-2-deactivate, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, GIT_CLEAN=True, ceph=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, org.label-schema.license=GPLv2) 2024-03-26T21:04:51.754 INFO:journalctl@ceph.osd.2.smithi063.stdout:Mar 26 21:04:51 smithi063 podman[77807]: 2024-03-26 21:04:51.717510241 +0000 UTC m=+0.635291073 container attach c2284be5ab1a0c30830f3c20b794eb5870ed604a3d99b8592bf3bc7953a92a0c (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-2-deactivate, GIT_CLEAN=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-03-26T21:04:52.006 INFO:journalctl@ceph.osd.2.smithi063.stdout:Mar 26 21:04:51 smithi063 podman[77807]: 2024-03-26 21:04:51.988624987 +0000 UTC m=+0.906405812 container died c2284be5ab1a0c30830f3c20b794eb5870ed604a3d99b8592bf3bc7953a92a0c (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-2-deactivate, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, org.label-schema.license=GPLv2, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-03-26T21:04:52.257 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:52 smithi063 ceph-mon[66934]: pgmap v139: 97 pgs: 6 peering, 3 stale+active+clean, 88 active+clean; 347 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4.4 KiB/s rd, 2.8 MiB/s wr, 322 op/s 2024-03-26T21:04:52.515 INFO:teuthology.orchestra.run.smithi063.stdout: 5 229622 10.56 MB/sec execute 406 sec latency 77.393 ms 2024-03-26T21:04:52.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:52 smithi174 ceph-mon[53014]: pgmap v139: 97 pgs: 6 peering, 3 stale+active+clean, 88 active+clean; 347 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4.4 KiB/s rd, 2.8 MiB/s wr, 322 op/s 2024-03-26T21:04:53.014 INFO:journalctl@ceph.osd.2.smithi063.stdout:Mar 26 21:04:52 smithi063 podman[77832]: 2024-03-26 21:04:52.954459577 +0000 UTC m=+0.952009256 container remove c2284be5ab1a0c30830f3c20b794eb5870ed604a3d99b8592bf3bc7953a92a0c (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-2-deactivate, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-26T21:04:53.014 INFO:journalctl@ceph.osd.2.smithi063.stdout:Mar 26 21:04:52 smithi063 systemd[1]: ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715@osd.2.service: Deactivated successfully. 2024-03-26T21:04:53.014 INFO:journalctl@ceph.osd.2.smithi063.stdout:Mar 26 21:04:52 smithi063 systemd[1]: Stopped Ceph osd.2 for 0ed4dd4c-ebb2-11ee-95d0-87774f69a715. 2024-03-26T21:04:53.014 INFO:journalctl@ceph.osd.2.smithi063.stdout:Mar 26 21:04:52 smithi063 systemd[1]: ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715@osd.2.service: Consumed 2min 44.992s CPU time. 2024-03-26T21:04:53.265 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:53 smithi063 ceph-mon[66934]: Health check failed: Degraded data redundancy: 108/1536 objects degraded (7.031%), 16 pgs degraded (PG_DEGRADED) 2024-03-26T21:04:53.514 INFO:teuthology.orchestra.run.smithi063.stdout: 5 229977 10.56 MB/sec execute 407 sec latency 38.728 ms 2024-03-26T21:04:53.517 INFO:journalctl@ceph.osd.2.smithi063.stdout:Mar 26 21:04:53 smithi063 systemd[1]: Starting Ceph osd.2 for 0ed4dd4c-ebb2-11ee-95d0-87774f69a715... 2024-03-26T21:04:53.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:53 smithi174 ceph-mon[53014]: Health check failed: Degraded data redundancy: 108/1536 objects degraded (7.031%), 16 pgs degraded (PG_DEGRADED) 2024-03-26T21:04:53.782 INFO:journalctl@ceph.osd.2.smithi063.stdout:Mar 26 21:04:53 smithi063 podman[77924]: 2024-03-26 21:04:53.551502355 +0000 UTC m=+0.028998517 image pull a9bdc3eaeb106993bcbbac35d7fe5c7b76c052021676e129f5984aed30aee063 quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d 2024-03-26T21:04:53.782 INFO:journalctl@ceph.osd.2.smithi063.stdout:Mar 26 21:04:53 smithi063 podman[77924]: 2024-03-26 21:04:53.706148019 +0000 UTC m=+0.183644167 container create 3f410c0e35737b98814fbfe47115c57f42cdbb24e6c2a11af39b6a2e31344c34 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-2-activate, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-03-26T21:04:54.038 INFO:journalctl@ceph.osd.2.smithi063.stdout:Mar 26 21:04:53 smithi063 podman[77924]: 2024-03-26 21:04:53.984401638 +0000 UTC m=+0.461897799 container init 3f410c0e35737b98814fbfe47115c57f42cdbb24e6c2a11af39b6a2e31344c34 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-2-activate, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, io.buildah.version=1.33.5, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-03-26T21:04:54.038 INFO:journalctl@ceph.osd.2.smithi063.stdout:Mar 26 21:04:53 smithi063 podman[77924]: 2024-03-26 21:04:53.992180682 +0000 UTC m=+0.469676831 container start 3f410c0e35737b98814fbfe47115c57f42cdbb24e6c2a11af39b6a2e31344c34 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-2-activate, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, GIT_CLEAN=True, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e) 2024-03-26T21:04:54.141 INFO:teuthology.orchestra.run.smithi063.stdout:true 2024-03-26T21:04:54.294 INFO:journalctl@ceph.osd.2.smithi063.stdout:Mar 26 21:04:54 smithi063 podman[77924]: 2024-03-26 21:04:54.110066317 +0000 UTC m=+0.587562470 container attach 3f410c0e35737b98814fbfe47115c57f42cdbb24e6c2a11af39b6a2e31344c34 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-2-activate, org.label-schema.license=GPLv2, org.label-schema.build-date=20240326, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-26T21:04:54.294 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:54 smithi063 ceph-mon[66934]: pgmap v140: 97 pgs: 16 active+undersized+degraded, 6 peering, 3 active+undersized, 72 active+clean; 338 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.2 MiB/s wr, 456 op/s; 108/1536 objects degraded (7.031%) 2024-03-26T21:04:54.514 INFO:teuthology.orchestra.run.smithi063.stdout: 5 230209 10.55 MB/sec execute 408 sec latency 48.315 ms 2024-03-26T21:04:54.550 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:54 smithi174 ceph-mon[53014]: pgmap v140: 97 pgs: 16 active+undersized+degraded, 6 peering, 3 active+undersized, 72 active+clean; 338 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.2 MiB/s wr, 456 op/s; 108/1536 objects degraded (7.031%) 2024-03-26T21:04:54.802 INFO:journalctl@ceph.osd.2.smithi063.stdout:Mar 26 21:04:54 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-2-activate[77961]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-03-26T21:04:54.802 INFO:journalctl@ceph.osd.2.smithi063.stdout:Mar 26 21:04:54 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-2-activate[77961]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-03-26T21:04:54.803 INFO:journalctl@ceph.osd.2.smithi063.stdout:Mar 26 21:04:54 smithi063 bash[77924]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-03-26T21:04:54.803 INFO:journalctl@ceph.osd.2.smithi063.stdout:Mar 26 21:04:54 smithi063 bash[77924]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-03-26T21:04:54.803 INFO:journalctl@ceph.osd.2.smithi063.stdout:Mar 26 21:04:54 smithi063 bash[77924]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-03-26T21:04:54.803 INFO:journalctl@ceph.osd.2.smithi063.stdout:Mar 26 21:04:54 smithi063 bash[77924]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-03-26T21:04:54.803 INFO:journalctl@ceph.osd.2.smithi063.stdout:Mar 26 21:04:54 smithi063 bash[77924]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-03-26T21:04:54.803 INFO:journalctl@ceph.osd.2.smithi063.stdout:Mar 26 21:04:54 smithi063 bash[77924]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-03-26T21:04:54.803 INFO:journalctl@ceph.osd.2.smithi063.stdout:Mar 26 21:04:54 smithi063 bash[77924]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-03-26T21:04:54.803 INFO:journalctl@ceph.osd.2.smithi063.stdout:Mar 26 21:04:54 smithi063 bash[77924]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-03-26T21:04:54.803 INFO:journalctl@ceph.osd.2.smithi063.stdout:Mar 26 21:04:54 smithi063 bash[77924]: --> ceph-volume raw activate successful for osd ID: 2 2024-03-26T21:04:54.803 INFO:journalctl@ceph.osd.2.smithi063.stdout:Mar 26 21:04:54 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-2-activate[77961]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-03-26T21:04:54.803 INFO:journalctl@ceph.osd.2.smithi063.stdout:Mar 26 21:04:54 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-2-activate[77961]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-03-26T21:04:54.803 INFO:journalctl@ceph.osd.2.smithi063.stdout:Mar 26 21:04:54 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-2-activate[77961]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-03-26T21:04:54.803 INFO:journalctl@ceph.osd.2.smithi063.stdout:Mar 26 21:04:54 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-2-activate[77961]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-03-26T21:04:54.803 INFO:journalctl@ceph.osd.2.smithi063.stdout:Mar 26 21:04:54 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-2-activate[77961]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-03-26T21:04:54.804 INFO:journalctl@ceph.osd.2.smithi063.stdout:Mar 26 21:04:54 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-2-activate[77961]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-03-26T21:04:54.804 INFO:journalctl@ceph.osd.2.smithi063.stdout:Mar 26 21:04:54 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-2-activate[77961]: --> ceph-volume raw activate successful for osd ID: 2 2024-03-26T21:04:54.804 INFO:journalctl@ceph.osd.2.smithi063.stdout:Mar 26 21:04:54 smithi063 podman[77924]: 2024-03-26 21:04:54.717404179 +0000 UTC m=+1.194900326 container died 3f410c0e35737b98814fbfe47115c57f42cdbb24e6c2a11af39b6a2e31344c34 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-2-activate, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, org.label-schema.license=GPLv2) 2024-03-26T21:04:55.206 INFO:teuthology.orchestra.run.smithi063.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-26T21:04:55.206 INFO:teuthology.orchestra.run.smithi063.stdout:alertmanager.smithi063 smithi063 *:9093,9094 running (6m) 36s ago 15m 24.5M - 0.23.0 ba2b418f427c 6e160c94aac3 2024-03-26T21:04:55.206 INFO:teuthology.orchestra.run.smithi063.stdout:crash.smithi063 smithi063 running (2m) 36s ago 15m 6665k - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 cf1096d84270 2024-03-26T21:04:55.206 INFO:teuthology.orchestra.run.smithi063.stdout:crash.smithi174 smithi174 running (2m) 2m ago 14m 6665k - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 cb5371a92684 2024-03-26T21:04:55.207 INFO:teuthology.orchestra.run.smithi063.stdout:grafana.smithi063 smithi063 *:3000 error 36s ago 14m - - 2024-03-26T21:04:55.207 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foofs.smithi063.tzaznz smithi063 running (9m) 36s ago 9m 37.9M - 17.2.0 e1d6a67b021e 10a989fc7954 2024-03-26T21:04:55.207 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foofs.smithi174.gitdvk smithi174 running (9m) 2m ago 9m 13.4M - 17.2.0 e1d6a67b021e 4339a5d98d1d 2024-03-26T21:04:55.207 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.smithi063.lxhujt smithi063 *:8443,9283,8765 running (5m) 36s ago 16m 505M - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 6d9d10330a90 2024-03-26T21:04:55.207 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.smithi174.sexoin smithi174 *:8443,9283,8765 running (4m) 2m ago 14m 450M - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 d7546233672d 2024-03-26T21:04:55.207 INFO:teuthology.orchestra.run.smithi063.stdout:mon.smithi063 smithi063 running (3m) 36s ago 16m 50.2M 2048M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 c644200344f8 2024-03-26T21:04:55.207 INFO:teuthology.orchestra.run.smithi063.stdout:mon.smithi174 smithi174 running (3m) 2m ago 14m 39.0M 2048M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 1a6d442692e7 2024-03-26T21:04:55.207 INFO:teuthology.orchestra.run.smithi063.stdout:nfs.foo.0.0.smithi063.fgmpwa smithi063 *:2049 running (9m) 36s ago 9m 277M - 4.0 e1d6a67b021e 3d6c0dd9ce51 2024-03-26T21:04:55.207 INFO:teuthology.orchestra.run.smithi063.stdout:nfs.foo.1.0.smithi174.pytptt smithi174 *:2049 running (8m) 2m ago 8m 86.4M - 4.0 e1d6a67b021e f220dbb449a3 2024-03-26T21:04:55.207 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.smithi063 smithi063 *:9100 running (5m) 36s ago 14m 13.9M - 1.3.1 1dbe0e931976 f52f41fd14c0 2024-03-26T21:04:55.207 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.smithi174 smithi174 *:9100 running (5m) 2m ago 14m 17.2M - 1.3.1 1dbe0e931976 0c027f39a4c8 2024-03-26T21:04:55.207 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0 smithi063 running (94s) 36s ago 13m 259M 1936M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 2d4b3fa825cf 2024-03-26T21:04:55.207 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1 smithi063 running (46s) 36s ago 13m 15.0M 1936M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 d41fbe507197 2024-03-26T21:04:55.207 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2 smithi063 running (12m) 36s ago 12m 499M 1936M 17.2.0 e1d6a67b021e 107f685ad08d 2024-03-26T21:04:55.207 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3 smithi063 running (12m) 36s ago 12m 442M 1936M 17.2.0 e1d6a67b021e ab0f1a855957 2024-03-26T21:04:55.208 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4 smithi174 running (11m) 2m ago 11m 575M 2704M 17.2.0 e1d6a67b021e a0e877ecc06d 2024-03-26T21:04:55.208 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5 smithi174 running (11m) 2m ago 11m 423M 2704M 17.2.0 e1d6a67b021e 378c2aca7e71 2024-03-26T21:04:55.208 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6 smithi174 running (11m) 2m ago 11m 328M 2704M 17.2.0 e1d6a67b021e ffbb5b7ad3b6 2024-03-26T21:04:55.208 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7 smithi174 running (10m) 2m ago 10m 524M 2704M 17.2.0 e1d6a67b021e c1d57fcabb3f 2024-03-26T21:04:55.208 INFO:teuthology.orchestra.run.smithi063.stdout:prometheus.smithi063 smithi063 *:9095 running (4m) 36s ago 14m 67.4M - 2.33.4 514e6a882f6e 567eef202627 2024-03-26T21:04:55.308 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:55 smithi063 ceph-mon[66934]: from='client.44209 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:04:55.514 INFO:teuthology.orchestra.run.smithi063.stdout: 5 230587 10.56 MB/sec execute 409 sec latency 53.862 ms 2024-03-26T21:04:55.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:55 smithi174 ceph-mon[53014]: from='client.44209 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:04:55.809 INFO:journalctl@ceph.osd.2.smithi063.stdout:Mar 26 21:04:55 smithi063 podman[78115]: 2024-03-26 21:04:55.652332195 +0000 UTC m=+0.927973491 container remove 3f410c0e35737b98814fbfe47115c57f42cdbb24e6c2a11af39b6a2e31344c34 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-2-activate, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5) 2024-03-26T21:04:55.838 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-03-26T21:04:55.839 INFO:teuthology.orchestra.run.smithi063.stdout: "mon": { 2024-03-26T21:04:55.839 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 2 2024-03-26T21:04:55.839 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:04:55.839 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr": { 2024-03-26T21:04:55.839 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 2 2024-03-26T21:04:55.839 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:04:55.839 INFO:teuthology.orchestra.run.smithi063.stdout: "osd": { 2024-03-26T21:04:55.839 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-03-26T21:04:55.839 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 2 2024-03-26T21:04:55.839 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:04:55.839 INFO:teuthology.orchestra.run.smithi063.stdout: "mds": { 2024-03-26T21:04:55.839 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-26T21:04:55.839 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:04:55.839 INFO:teuthology.orchestra.run.smithi063.stdout: "overall": { 2024-03-26T21:04:55.840 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-03-26T21:04:55.840 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 6 2024-03-26T21:04:55.840 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-03-26T21:04:55.840 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-03-26T21:04:56.066 INFO:journalctl@ceph.osd.2.smithi063.stdout:Mar 26 21:04:55 smithi063 podman[78207]: 2024-03-26 21:04:55.845308515 +0000 UTC m=+0.031579996 image pull a9bdc3eaeb106993bcbbac35d7fe5c7b76c052021676e129f5984aed30aee063 quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d 2024-03-26T21:04:56.067 INFO:journalctl@ceph.osd.2.smithi063.stdout:Mar 26 21:04:55 smithi063 podman[78207]: 2024-03-26 21:04:55.978958865 +0000 UTC m=+0.165230354 container create f123eeabd3fa7d5076b1f7427d92ae5367f7aea4ec0eb6aa34678d6417edafaf (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-2, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240326, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-03-26T21:04:56.318 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:56 smithi063 ceph-mon[66934]: from='client.44213 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:04:56.318 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:56 smithi063 ceph-mon[66934]: pgmap v141: 97 pgs: 19 active+undersized+degraded, 6 active+undersized, 72 active+clean; 338 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 3.4 KiB/s rd, 2.6 MiB/s wr, 271 op/s; 146/1551 objects degraded (9.413%) 2024-03-26T21:04:56.318 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:56 smithi063 ceph-mon[66934]: from='client.44217 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:04:56.318 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:56 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:04:56.318 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:56 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:04:56.318 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:56 smithi063 ceph-mon[66934]: from='client.? 172.21.15.63:0/2830066794' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:04:56.373 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-03-26T21:04:56.373 INFO:teuthology.orchestra.run.smithi063.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d", 2024-03-26T21:04:56.373 INFO:teuthology.orchestra.run.smithi063.stdout: "in_progress": true, 2024-03-26T21:04:56.373 INFO:teuthology.orchestra.run.smithi063.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-26T21:04:56.374 INFO:teuthology.orchestra.run.smithi063.stdout: "services_complete": [ 2024-03-26T21:04:56.374 INFO:teuthology.orchestra.run.smithi063.stdout: "crash", 2024-03-26T21:04:56.374 INFO:teuthology.orchestra.run.smithi063.stdout: "mon", 2024-03-26T21:04:56.374 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr" 2024-03-26T21:04:56.374 INFO:teuthology.orchestra.run.smithi063.stdout: ], 2024-03-26T21:04:56.374 INFO:teuthology.orchestra.run.smithi063.stdout: "progress": "8/23 daemons upgraded", 2024-03-26T21:04:56.374 INFO:teuthology.orchestra.run.smithi063.stdout: "message": "Currently upgrading osd daemons", 2024-03-26T21:04:56.374 INFO:teuthology.orchestra.run.smithi063.stdout: "is_paused": false 2024-03-26T21:04:56.374 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-03-26T21:04:56.515 INFO:teuthology.orchestra.run.smithi063.stdout: 5 230927 10.56 MB/sec execute 410 sec latency 38.702 ms 2024-03-26T21:04:56.550 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:56 smithi174 ceph-mon[53014]: from='client.44213 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:04:56.550 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:56 smithi174 ceph-mon[53014]: pgmap v141: 97 pgs: 19 active+undersized+degraded, 6 active+undersized, 72 active+clean; 338 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 3.4 KiB/s rd, 2.6 MiB/s wr, 271 op/s; 146/1551 objects degraded (9.413%) 2024-03-26T21:04:56.550 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:56 smithi174 ceph-mon[53014]: from='client.44217 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:04:56.550 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:56 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:04:56.550 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:56 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:04:56.550 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:56 smithi174 ceph-mon[53014]: from='client.? 172.21.15.63:0/2830066794' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:04:56.581 INFO:journalctl@ceph.osd.2.smithi063.stdout:Mar 26 21:04:56 smithi063 podman[78207]: 2024-03-26 21:04:56.374542097 +0000 UTC m=+0.560813580 container init f123eeabd3fa7d5076b1f7427d92ae5367f7aea4ec0eb6aa34678d6417edafaf (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-2, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux ) 2024-03-26T21:04:56.582 INFO:journalctl@ceph.osd.2.smithi063.stdout:Mar 26 21:04:56 smithi063 podman[78207]: 2024-03-26 21:04:56.386113718 +0000 UTC m=+0.572385202 container start f123eeabd3fa7d5076b1f7427d92ae5367f7aea4ec0eb6aa34678d6417edafaf (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-2, ceph=True, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, GIT_CLEAN=True, org.label-schema.build-date=20240326, io.buildah.version=1.33.5) 2024-03-26T21:04:56.582 INFO:journalctl@ceph.osd.2.smithi063.stdout:Mar 26 21:04:56 smithi063 ceph-osd[78250]: -- 172.21.15.63:0/2631173906 <== mon.0 v2:172.21.15.63:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x556729df1680 con 0x556729dbac00 2024-03-26T21:04:56.582 INFO:journalctl@ceph.osd.2.smithi063.stdout:Mar 26 21:04:56 smithi063 bash[78207]: f123eeabd3fa7d5076b1f7427d92ae5367f7aea4ec0eb6aa34678d6417edafaf 2024-03-26T21:04:56.582 INFO:journalctl@ceph.osd.2.smithi063.stdout:Mar 26 21:04:56 smithi063 systemd[1]: Started Ceph osd.2 for 0ed4dd4c-ebb2-11ee-95d0-87774f69a715. 2024-03-26T21:04:57.083 INFO:journalctl@ceph.osd.2.smithi063.stdout:Mar 26 21:04:56 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-2[78244]: 2024-03-26T21:04:56.973+0000 7fa9c92a27c0 -1 Falling back to public interface 2024-03-26T21:04:57.130 INFO:teuthology.orchestra.run.smithi063.stdout:HEALTH_WARN 1 failed cephadm daemon(s); 1 osds down; Degraded data redundancy: 146/1551 objects degraded (9.413%), 19 pgs degraded 2024-03-26T21:04:57.130 INFO:teuthology.orchestra.run.smithi063.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-26T21:04:57.130 INFO:teuthology.orchestra.run.smithi063.stdout: daemon grafana.smithi063 on smithi063 is in error state 2024-03-26T21:04:57.131 INFO:teuthology.orchestra.run.smithi063.stdout:[WRN] OSD_DOWN: 1 osds down 2024-03-26T21:04:57.131 INFO:teuthology.orchestra.run.smithi063.stdout: osd.2 (root=default,host=smithi063) is down 2024-03-26T21:04:57.131 INFO:teuthology.orchestra.run.smithi063.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 146/1551 objects degraded (9.413%), 19 pgs degraded 2024-03-26T21:04:57.131 INFO:teuthology.orchestra.run.smithi063.stdout: pg 2.1 is active+undersized+degraded, acting [3,0] 2024-03-26T21:04:57.131 INFO:teuthology.orchestra.run.smithi063.stdout: pg 2.3 is active+undersized+degraded, acting [5,4] 2024-03-26T21:04:57.131 INFO:teuthology.orchestra.run.smithi063.stdout: pg 2.7 is active+undersized+degraded, acting [6,7] 2024-03-26T21:04:57.131 INFO:teuthology.orchestra.run.smithi063.stdout: pg 2.c is active+undersized+degraded, acting [5,0] 2024-03-26T21:04:57.131 INFO:teuthology.orchestra.run.smithi063.stdout: pg 2.e is active+undersized+degraded, acting [3,7] 2024-03-26T21:04:57.131 INFO:teuthology.orchestra.run.smithi063.stdout: pg 2.10 is active+undersized+degraded, acting [1,0] 2024-03-26T21:04:57.131 INFO:teuthology.orchestra.run.smithi063.stdout: pg 2.16 is active+undersized+degraded, acting [5,6] 2024-03-26T21:04:57.131 INFO:teuthology.orchestra.run.smithi063.stdout: pg 2.17 is active+undersized+degraded, acting [6,5] 2024-03-26T21:04:57.131 INFO:teuthology.orchestra.run.smithi063.stdout: pg 2.1c is active+undersized+degraded, acting [7,5] 2024-03-26T21:04:57.131 INFO:teuthology.orchestra.run.smithi063.stdout: pg 3.0 is active+undersized+degraded, acting [1,6] 2024-03-26T21:04:57.131 INFO:teuthology.orchestra.run.smithi063.stdout: pg 3.4 is active+undersized+degraded, acting [1,5] 2024-03-26T21:04:57.132 INFO:teuthology.orchestra.run.smithi063.stdout: pg 3.5 is active+undersized+degraded, acting [5,3] 2024-03-26T21:04:57.132 INFO:teuthology.orchestra.run.smithi063.stdout: pg 3.9 is active+undersized+degraded, acting [4,7] 2024-03-26T21:04:57.132 INFO:teuthology.orchestra.run.smithi063.stdout: pg 3.13 is active+undersized+degraded, acting [7,4] 2024-03-26T21:04:57.132 INFO:teuthology.orchestra.run.smithi063.stdout: pg 3.1a is active+undersized+degraded, acting [4,1] 2024-03-26T21:04:57.132 INFO:teuthology.orchestra.run.smithi063.stdout: pg 3.1e is active+undersized+degraded, acting [3,6] 2024-03-26T21:04:57.132 INFO:teuthology.orchestra.run.smithi063.stdout: pg 3.1f is active+undersized+degraded, acting [0,5] 2024-03-26T21:04:57.132 INFO:teuthology.orchestra.run.smithi063.stdout: pg 4.14 is active+undersized+degraded, acting [3,7] 2024-03-26T21:04:57.132 INFO:teuthology.orchestra.run.smithi063.stdout: pg 4.19 is active+undersized+degraded, acting [3,0] 2024-03-26T21:04:57.514 INFO:teuthology.orchestra.run.smithi063.stdout: 5 231278 10.56 MB/sec execute 411 sec latency 36.451 ms 2024-03-26T21:04:57.845 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:57 smithi063 ceph-mon[66934]: from='client.34296 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:04:57.845 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:57 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:04:57.845 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:57 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:04:57.845 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:57 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:04:57.845 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:57 smithi063 ceph-mon[66934]: pgmap v142: 97 pgs: 19 active+undersized+degraded, 6 active+undersized, 72 active+clean; 341 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.3 MiB/s wr, 430 op/s; 142/1626 objects degraded (8.733%) 2024-03-26T21:04:57.845 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:57 smithi063 ceph-mon[66934]: from='client.? 172.21.15.63:0/3502642033' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-26T21:04:58.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:57 smithi174 ceph-mon[53014]: from='client.34296 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:04:58.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:57 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:04:58.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:57 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:04:58.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:57 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:04:58.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:57 smithi174 ceph-mon[53014]: pgmap v142: 97 pgs: 19 active+undersized+degraded, 6 active+undersized, 72 active+clean; 341 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.3 MiB/s wr, 430 op/s; 142/1626 objects degraded (8.733%) 2024-03-26T21:04:58.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:57 smithi174 ceph-mon[53014]: from='client.? 172.21.15.63:0/3502642033' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-26T21:04:58.515 INFO:teuthology.orchestra.run.smithi063.stdout: 5 231602 10.54 MB/sec execute 412 sec latency 44.427 ms 2024-03-26T21:04:59.121 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:58 smithi063 ceph-mon[66934]: Health check update: Degraded data redundancy: 144/1650 objects degraded (8.727%), 19 pgs degraded (PG_DEGRADED) 2024-03-26T21:04:59.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:58 smithi174 ceph-mon[53014]: Health check update: Degraded data redundancy: 144/1650 objects degraded (8.727%), 19 pgs degraded (PG_DEGRADED) 2024-03-26T21:04:59.514 INFO:teuthology.orchestra.run.smithi063.stdout: 5 231873 10.53 MB/sec execute 413 sec latency 43.668 ms 2024-03-26T21:05:00.134 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:04:59 smithi063 ceph-mon[66934]: pgmap v143: 97 pgs: 19 active+undersized+degraded, 6 active+undersized, 72 active+clean; 342 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.3 KiB/s rd, 4.1 MiB/s wr, 369 op/s; 144/1650 objects degraded (8.727%) 2024-03-26T21:05:00.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:04:59 smithi174 ceph-mon[53014]: pgmap v143: 97 pgs: 19 active+undersized+degraded, 6 active+undersized, 72 active+clean; 342 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.3 KiB/s rd, 4.1 MiB/s wr, 369 op/s; 144/1650 objects degraded (8.727%) 2024-03-26T21:05:00.514 INFO:teuthology.orchestra.run.smithi063.stdout: 5 232197 10.52 MB/sec execute 414 sec latency 63.278 ms 2024-03-26T21:05:01.515 INFO:teuthology.orchestra.run.smithi063.stdout: 5 232502 10.52 MB/sec execute 415 sec latency 47.555 ms 2024-03-26T21:05:01.939 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:01 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:05:01.940 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:01 smithi063 ceph-mon[66934]: pgmap v144: 97 pgs: 19 active+undersized+degraded, 6 active+undersized, 72 active+clean; 350 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 4.8 KiB/s rd, 4.5 MiB/s wr, 495 op/s; 143/1659 objects degraded (8.620%) 2024-03-26T21:05:02.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:01 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:05:02.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:01 smithi174 ceph-mon[53014]: pgmap v144: 97 pgs: 19 active+undersized+degraded, 6 active+undersized, 72 active+clean; 350 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 4.8 KiB/s rd, 4.5 MiB/s wr, 495 op/s; 143/1659 objects degraded (8.620%) 2024-03-26T21:05:02.515 INFO:teuthology.orchestra.run.smithi063.stdout: 5 232951 10.53 MB/sec execute 416 sec latency 38.922 ms 2024-03-26T21:05:03.204 INFO:journalctl@ceph.osd.2.smithi063.stdout:Mar 26 21:05:03 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-2[78244]: 2024-03-26T21:05:03.192+0000 7fa9c92a27c0 -1 osd.2 0 read_superblock omap replica is missing. 2024-03-26T21:05:03.514 INFO:teuthology.orchestra.run.smithi063.stdout: 5 233294 10.52 MB/sec execute 417 sec latency 82.585 ms 2024-03-26T21:05:03.965 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:03 smithi063 ceph-mon[66934]: pgmap v145: 97 pgs: 19 active+undersized+degraded, 6 active+undersized, 72 active+clean; 342 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.3 MiB/s wr, 528 op/s; 141/1584 objects degraded (8.902%) 2024-03-26T21:05:04.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:03 smithi174 ceph-mon[53014]: pgmap v145: 97 pgs: 19 active+undersized+degraded, 6 active+undersized, 72 active+clean; 342 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.3 MiB/s wr, 528 op/s; 141/1584 objects degraded (8.902%) 2024-03-26T21:05:04.478 INFO:journalctl@ceph.osd.2.smithi063.stdout:Mar 26 21:05:04 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-2[78244]: 2024-03-26T21:05:04.271+0000 7fa9c92a27c0 -1 osd.2 97 log_to_monitors true 2024-03-26T21:05:04.514 INFO:teuthology.orchestra.run.smithi063.stdout: 5 233668 10.52 MB/sec execute 418 sec latency 38.189 ms 2024-03-26T21:05:04.987 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:04 smithi063 ceph-mon[66934]: from='osd.2 [v2:172.21.15.63:6818/3040449353,v1:172.21.15.63:6819/3040449353]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-03-26T21:05:04.987 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:04 smithi063 ceph-mon[66934]: Health check update: Degraded data redundancy: 141/1584 objects degraded (8.902%), 19 pgs degraded (PG_DEGRADED) 2024-03-26T21:05:05.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:04 smithi174 ceph-mon[53014]: from='osd.2 [v2:172.21.15.63:6818/3040449353,v1:172.21.15.63:6819/3040449353]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-03-26T21:05:05.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:04 smithi174 ceph-mon[53014]: Health check update: Degraded data redundancy: 141/1584 objects degraded (8.902%), 19 pgs degraded (PG_DEGRADED) 2024-03-26T21:05:05.494 INFO:journalctl@ceph.osd.2.smithi063.stdout:Mar 26 21:05:05 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-2[78244]: 2024-03-26T21:05:05.432+0000 7fa9b66f2700 -1 osd.2 97 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-26T21:05:05.514 INFO:teuthology.orchestra.run.smithi063.stdout: 5 233851 10.51 MB/sec execute 419 sec latency 161.888 ms 2024-03-26T21:05:06.008 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:05 smithi063 ceph-mon[66934]: pgmap v146: 97 pgs: 19 active+undersized+degraded, 6 active+undersized, 72 active+clean; 342 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.2 MiB/s wr, 409 op/s; 138/1569 objects degraded (8.795%) 2024-03-26T21:05:06.008 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:05 smithi063 ceph-mon[66934]: from='osd.2 [v2:172.21.15.63:6818/3040449353,v1:172.21.15.63:6819/3040449353]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-03-26T21:05:06.008 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:05 smithi063 ceph-mon[66934]: osdmap e100: 8 total, 7 up, 8 in 2024-03-26T21:05:06.008 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:05 smithi063 ceph-mon[66934]: from='osd.2 [v2:172.21.15.63:6818/3040449353,v1:172.21.15.63:6819/3040449353]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi063", "root=default"]}]: dispatch 2024-03-26T21:05:06.008 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:05 smithi063 ceph-mon[66934]: from='osd.2 [v2:172.21.15.63:6818/3040449353,v1:172.21.15.63:6819/3040449353]' entity='osd.2' 2024-03-26T21:05:06.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:05 smithi174 ceph-mon[53014]: pgmap v146: 97 pgs: 19 active+undersized+degraded, 6 active+undersized, 72 active+clean; 342 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.2 MiB/s wr, 409 op/s; 138/1569 objects degraded (8.795%) 2024-03-26T21:05:06.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:05 smithi174 ceph-mon[53014]: from='osd.2 [v2:172.21.15.63:6818/3040449353,v1:172.21.15.63:6819/3040449353]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-03-26T21:05:06.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:05 smithi174 ceph-mon[53014]: osdmap e100: 8 total, 7 up, 8 in 2024-03-26T21:05:06.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:05 smithi174 ceph-mon[53014]: from='osd.2 [v2:172.21.15.63:6818/3040449353,v1:172.21.15.63:6819/3040449353]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi063", "root=default"]}]: dispatch 2024-03-26T21:05:06.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:05 smithi174 ceph-mon[53014]: from='osd.2 [v2:172.21.15.63:6818/3040449353,v1:172.21.15.63:6819/3040449353]' entity='osd.2' 2024-03-26T21:05:06.514 INFO:teuthology.orchestra.run.smithi063.stdout: 5 234171 10.52 MB/sec execute 420 sec latency 45.352 ms 2024-03-26T21:05:07.515 INFO:teuthology.orchestra.run.smithi063.stdout: 5 234392 10.50 MB/sec execute 421 sec latency 728.322 ms 2024-03-26T21:05:07.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:07 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:05:07.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:07 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:05:07.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:07 smithi174 ceph-mon[53014]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-26T21:05:07.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:07 smithi174 ceph-mon[53014]: osd.2 [v2:172.21.15.63:6818/3040449353,v1:172.21.15.63:6819/3040449353] boot 2024-03-26T21:05:07.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:07 smithi174 ceph-mon[53014]: osdmap e101: 8 total, 8 up, 8 in 2024-03-26T21:05:07.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:07 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-26T21:05:07.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:07 smithi174 ceph-mon[53014]: pgmap v149: 97 pgs: 14 peering, 7 active+undersized+degraded, 4 active+undersized, 72 active+clean; 350 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 5.8 MiB/s wr, 665 op/s; 60/1635 objects degraded (3.670%) 2024-03-26T21:05:07.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:07 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:05:07.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:07 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:05:07.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:07 smithi063 ceph-mon[66934]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-26T21:05:07.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:07 smithi063 ceph-mon[66934]: osd.2 [v2:172.21.15.63:6818/3040449353,v1:172.21.15.63:6819/3040449353] boot 2024-03-26T21:05:07.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:07 smithi063 ceph-mon[66934]: osdmap e101: 8 total, 8 up, 8 in 2024-03-26T21:05:07.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:07 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-26T21:05:07.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:07 smithi063 ceph-mon[66934]: pgmap v149: 97 pgs: 14 peering, 7 active+undersized+degraded, 4 active+undersized, 72 active+clean; 350 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 5.8 MiB/s wr, 665 op/s; 60/1635 objects degraded (3.670%) 2024-03-26T21:05:08.515 INFO:teuthology.orchestra.run.smithi063.stdout: 5 234401 10.48 MB/sec execute 422 sec latency 1658.527 ms 2024-03-26T21:05:08.619 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:08 smithi063 ceph-mon[66934]: osdmap e102: 8 total, 8 up, 8 in 2024-03-26T21:05:08.619 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:08 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:05:08.619 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:08 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:05:08.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:08 smithi174 ceph-mon[53014]: osdmap e102: 8 total, 8 up, 8 in 2024-03-26T21:05:08.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:08 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:05:08.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:08 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:05:09.515 INFO:teuthology.orchestra.run.smithi063.stdout: 5 234703 10.48 MB/sec execute 423 sec latency 67.922 ms 2024-03-26T21:05:09.638 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:09 smithi063 ceph-mon[66934]: osdmap e103: 8 total, 8 up, 8 in 2024-03-26T21:05:09.638 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:09 smithi063 ceph-mon[66934]: pgmap v152: 97 pgs: 16 peering, 6 active+undersized+degraded, 3 active+undersized, 72 active+clean; 351 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.8 MiB/s wr, 558 op/s; 48/1680 objects degraded (2.857%) 2024-03-26T21:05:09.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:09 smithi174 ceph-mon[53014]: osdmap e103: 8 total, 8 up, 8 in 2024-03-26T21:05:09.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:09 smithi174 ceph-mon[53014]: pgmap v152: 97 pgs: 16 peering, 6 active+undersized+degraded, 3 active+undersized, 72 active+clean; 351 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.8 MiB/s wr, 558 op/s; 48/1680 objects degraded (2.857%) 2024-03-26T21:05:10.515 INFO:teuthology.orchestra.run.smithi063.stdout: 5 234931 10.46 MB/sec execute 424 sec latency 704.836 ms 2024-03-26T21:05:10.665 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:10 smithi063 ceph-mon[66934]: osdmap e104: 8 total, 8 up, 8 in 2024-03-26T21:05:10.665 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:10 smithi063 ceph-mon[66934]: Health check update: Degraded data redundancy: 48/1680 objects degraded (2.857%), 6 pgs degraded (PG_DEGRADED) 2024-03-26T21:05:10.665 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:10 smithi063 ceph-mon[66934]: osdmap e105: 8 total, 8 up, 8 in 2024-03-26T21:05:10.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:10 smithi174 ceph-mon[53014]: osdmap e104: 8 total, 8 up, 8 in 2024-03-26T21:05:10.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:10 smithi174 ceph-mon[53014]: Health check update: Degraded data redundancy: 48/1680 objects degraded (2.857%), 6 pgs degraded (PG_DEGRADED) 2024-03-26T21:05:10.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:10 smithi174 ceph-mon[53014]: osdmap e105: 8 total, 8 up, 8 in 2024-03-26T21:05:11.515 INFO:teuthology.orchestra.run.smithi063.stdout: 5 235281 10.45 MB/sec execute 425 sec latency 811.740 ms 2024-03-26T21:05:11.699 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:11 smithi063 ceph-mon[66934]: osdmap e106: 8 total, 8 up, 8 in 2024-03-26T21:05:11.699 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:11 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:05:11.699 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:11 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:05:11.699 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:11 smithi063 ceph-mon[66934]: pgmap v156: 97 pgs: 1 active+undersized+remapped, 16 peering, 80 active+clean; 348 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.9 KiB/s rd, 508 KiB/s wr, 31 op/s; 1.3 MiB/s, 13 objects/s recovering 2024-03-26T21:05:12.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:11 smithi174 ceph-mon[53014]: osdmap e106: 8 total, 8 up, 8 in 2024-03-26T21:05:12.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:11 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:05:12.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:11 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:05:12.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:11 smithi174 ceph-mon[53014]: pgmap v156: 97 pgs: 1 active+undersized+remapped, 16 peering, 80 active+clean; 348 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.9 KiB/s rd, 508 KiB/s wr, 31 op/s; 1.3 MiB/s, 13 objects/s recovering 2024-03-26T21:05:12.515 INFO:teuthology.orchestra.run.smithi063.stdout: 5 235702 10.45 MB/sec execute 426 sec latency 155.720 ms 2024-03-26T21:05:12.761 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:12 smithi063 ceph-mon[66934]: osdmap e107: 8 total, 8 up, 8 in 2024-03-26T21:05:12.762 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:12 smithi063 ceph-mon[66934]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 48/1680 objects degraded (2.857%), 6 pgs degraded) 2024-03-26T21:05:13.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:12 smithi174 ceph-mon[53014]: osdmap e107: 8 total, 8 up, 8 in 2024-03-26T21:05:13.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:12 smithi174 ceph-mon[53014]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 48/1680 objects degraded (2.857%), 6 pgs degraded) 2024-03-26T21:05:13.515 INFO:teuthology.orchestra.run.smithi063.stdout: 5 236110 10.45 MB/sec execute 427 sec latency 53.263 ms 2024-03-26T21:05:13.778 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:13 smithi063 ceph-mon[66934]: pgmap v158: 97 pgs: 1 active+undersized+remapped, 96 active+clean; 340 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 4.2 MiB/s wr, 435 op/s; 2.2 MiB/s, 38 objects/s recovering 2024-03-26T21:05:13.778 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:13 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:05:13.778 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:13 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:05:13.778 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:13 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:05:13.778 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:13 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T21:05:13.778 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:13 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:05:13.778 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:13 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:05:14.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:13 smithi174 ceph-mon[53014]: pgmap v158: 97 pgs: 1 active+undersized+remapped, 96 active+clean; 340 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 4.2 MiB/s wr, 435 op/s; 2.2 MiB/s, 38 objects/s recovering 2024-03-26T21:05:14.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:13 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:05:14.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:13 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:05:14.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:13 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:05:14.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:13 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T21:05:14.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:13 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:05:14.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:13 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:05:14.515 INFO:teuthology.orchestra.run.smithi063.stdout: 5 236477 10.45 MB/sec execute 428 sec latency 54.623 ms 2024-03-26T21:05:14.810 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:14 smithi063 ceph-mon[66934]: pgmap v159: 97 pgs: 1 active+undersized+remapped, 96 active+clean; 340 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 4.2 MiB/s wr, 429 op/s; 2.1 MiB/s, 37 objects/s recovering 2024-03-26T21:05:14.810 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:14 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:05:14.810 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:14 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:05:14.810 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:14 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:05:14.810 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:14 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:05:14.810 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:14 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-03-26T21:05:14.810 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:14 smithi063 ceph-mon[66934]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-03-26T21:05:14.810 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:14 smithi063 ceph-mon[66934]: Upgrade: osd.3 is safe to restart 2024-03-26T21:05:15.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:14 smithi174 ceph-mon[53014]: pgmap v159: 97 pgs: 1 active+undersized+remapped, 96 active+clean; 340 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 4.2 MiB/s wr, 429 op/s; 2.1 MiB/s, 37 objects/s recovering 2024-03-26T21:05:15.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:14 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:05:15.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:14 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:05:15.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:14 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:05:15.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:14 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:05:15.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:14 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-03-26T21:05:15.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:14 smithi174 ceph-mon[53014]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-03-26T21:05:15.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:14 smithi174 ceph-mon[53014]: Upgrade: osd.3 is safe to restart 2024-03-26T21:05:15.515 INFO:teuthology.orchestra.run.smithi063.stdout: 5 236876 10.46 MB/sec execute 429 sec latency 69.162 ms 2024-03-26T21:05:16.515 INFO:teuthology.orchestra.run.smithi063.stdout: 5 237295 10.46 MB/sec execute 430 sec latency 41.258 ms 2024-03-26T21:05:17.341 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:17 smithi063 ceph-mon[66934]: pgmap v160: 97 pgs: 97 active+clean; 342 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 3.6 MiB/s wr, 461 op/s; 1.5 MiB/s, 26 objects/s recovering 2024-03-26T21:05:17.341 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:17 smithi063 ceph-mon[66934]: Upgrade: Updating osd.3 2024-03-26T21:05:17.341 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:17 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:05:17.341 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:17 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-03-26T21:05:17.341 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:17 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:05:17.341 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:17 smithi063 ceph-mon[66934]: Deploying daemon osd.3 on smithi063 2024-03-26T21:05:17.515 INFO:teuthology.orchestra.run.smithi063.stdout: 5 237642 10.46 MB/sec execute 431 sec latency 68.489 ms 2024-03-26T21:05:17.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:17 smithi174 ceph-mon[53014]: pgmap v160: 97 pgs: 97 active+clean; 342 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 3.6 MiB/s wr, 461 op/s; 1.5 MiB/s, 26 objects/s recovering 2024-03-26T21:05:17.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:17 smithi174 ceph-mon[53014]: Upgrade: Updating osd.3 2024-03-26T21:05:17.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:17 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:05:17.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:17 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-03-26T21:05:17.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:17 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:05:17.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:17 smithi174 ceph-mon[53014]: Deploying daemon osd.3 on smithi063 2024-03-26T21:05:18.515 INFO:teuthology.orchestra.run.smithi063.stdout: 5 238035 10.46 MB/sec execute 432 sec latency 34.524 ms 2024-03-26T21:05:19.398 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:19 smithi063 ceph-mon[66934]: pgmap v161: 97 pgs: 97 active+clean; 343 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 6.0 MiB/s wr, 566 op/s; 625 KiB/s, 15 objects/s recovering 2024-03-26T21:05:19.515 INFO:teuthology.orchestra.run.smithi063.stdout: 5 238379 10.46 MB/sec execute 433 sec latency 50.790 ms 2024-03-26T21:05:19.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:19 smithi174 ceph-mon[53014]: pgmap v161: 97 pgs: 97 active+clean; 343 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 6.0 MiB/s wr, 566 op/s; 625 KiB/s, 15 objects/s recovering 2024-03-26T21:05:20.515 INFO:teuthology.orchestra.run.smithi063.stdout: 5 238651 10.45 MB/sec execute 434 sec latency 81.947 ms 2024-03-26T21:05:21.254 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:21 smithi063 ceph-mon[66934]: pgmap v162: 97 pgs: 97 active+clean; 343 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 4.7 MiB/s wr, 450 op/s; 497 KiB/s, 12 objects/s recovering 2024-03-26T21:05:21.254 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:21 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:05:21.515 INFO:teuthology.orchestra.run.smithi063.stdout: 5 238946 10.44 MB/sec execute 435 sec latency 53.898 ms 2024-03-26T21:05:21.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:21 smithi174 ceph-mon[53014]: pgmap v162: 97 pgs: 97 active+clean; 343 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 4.7 MiB/s wr, 450 op/s; 497 KiB/s, 12 objects/s recovering 2024-03-26T21:05:21.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:21 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:05:21.761 INFO:journalctl@ceph.osd.3.smithi063.stdout:Mar 26 21:05:21 smithi063 systemd[1]: Stopping Ceph osd.3 for 0ed4dd4c-ebb2-11ee-95d0-87774f69a715... 2024-03-26T21:05:22.019 INFO:journalctl@ceph.osd.3.smithi063.stdout:Mar 26 21:05:21 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-3[46345]: 2024-03-26T21:05:21.994+0000 7f28d8fa1700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-26T21:05:22.020 INFO:journalctl@ceph.osd.3.smithi063.stdout:Mar 26 21:05:21 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-3[46345]: 2024-03-26T21:05:21.994+0000 7f28d8fa1700 -1 osd.3 107 *** Got signal Terminated *** 2024-03-26T21:05:22.020 INFO:journalctl@ceph.osd.3.smithi063.stdout:Mar 26 21:05:21 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-3[46345]: 2024-03-26T21:05:21.994+0000 7f28d8fa1700 -1 osd.3 107 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-26T21:05:22.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:22 smithi063 ceph-mon[66934]: osd.3 marked itself down and dead 2024-03-26T21:05:22.515 INFO:teuthology.orchestra.run.smithi063.stdout: 5 239155 10.42 MB/sec execute 436 sec latency 297.049 ms 2024-03-26T21:05:22.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:22 smithi174 ceph-mon[53014]: osd.3 marked itself down and dead 2024-03-26T21:05:23.230 INFO:journalctl@ceph.osd.3.smithi063.stdout:Mar 26 21:05:22 smithi063 podman[80232]: 2024-03-26 21:05:22.927895988 +0000 UTC m=+1.058912613 container died ab0f1a855957a7ad65577d62b0698ef173a0c2a12be1d9767e080eabda78d478 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-3, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , version=8, io.openshift.expose-services=, vendor=Red Hat, Inc., vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, architecture=x86_64, release=754, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-03-26T21:05:23.484 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:23 smithi063 ceph-mon[66934]: pgmap v163: 97 pgs: 97 active+clean; 345 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 8.1 KiB/s rd, 4.9 MiB/s wr, 543 op/s; 436 KiB/s, 11 objects/s recovering 2024-03-26T21:05:23.484 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:23 smithi063 ceph-mon[66934]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-26T21:05:23.484 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:23 smithi063 ceph-mon[66934]: osdmap e108: 8 total, 7 up, 8 in 2024-03-26T21:05:23.484 INFO:journalctl@ceph.osd.3.smithi063.stdout:Mar 26 21:05:23 smithi063 podman[80232]: 2024-03-26 21:05:23.303515787 +0000 UTC m=+1.434532412 container cleanup ab0f1a855957a7ad65577d62b0698ef173a0c2a12be1d9767e080eabda78d478 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-3, version=8, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, distribution-scope=public, maintainer=Guillaume Abrioux , GIT_CLEAN=True, vcs-type=git, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64) 2024-03-26T21:05:23.484 INFO:journalctl@ceph.osd.3.smithi063.stdout:Mar 26 21:05:23 smithi063 bash[80232]: ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-3 2024-03-26T21:05:23.515 INFO:teuthology.orchestra.run.smithi063.stdout: 5 239248 10.40 MB/sec execute 437 sec latency 1067.276 ms 2024-03-26T21:05:23.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:23 smithi174 ceph-mon[53014]: pgmap v163: 97 pgs: 97 active+clean; 345 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 8.1 KiB/s rd, 4.9 MiB/s wr, 543 op/s; 436 KiB/s, 11 objects/s recovering 2024-03-26T21:05:23.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:23 smithi174 ceph-mon[53014]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-26T21:05:23.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:23 smithi174 ceph-mon[53014]: osdmap e108: 8 total, 7 up, 8 in 2024-03-26T21:05:23.740 INFO:journalctl@ceph.osd.3.smithi063.stdout:Mar 26 21:05:23 smithi063 podman[80250]: 2024-03-26 21:05:23.539130348 +0000 UTC m=+0.606582415 container remove ab0f1a855957a7ad65577d62b0698ef173a0c2a12be1d9767e080eabda78d478 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-3, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, name=centos-stream, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, GIT_BRANCH=HEAD, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True) 2024-03-26T21:05:24.247 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:24 smithi063 ceph-mon[66934]: osdmap e109: 8 total, 7 up, 8 in 2024-03-26T21:05:24.247 INFO:journalctl@ceph.osd.3.smithi063.stdout:Mar 26 21:05:24 smithi063 podman[80329]: 2024-03-26 21:05:23.984432697 +0000 UTC m=+0.034012678 image pull a9bdc3eaeb106993bcbbac35d7fe5c7b76c052021676e129f5984aed30aee063 quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d 2024-03-26T21:05:24.247 INFO:journalctl@ceph.osd.3.smithi063.stdout:Mar 26 21:05:24 smithi063 podman[80329]: 2024-03-26 21:05:24.157588578 +0000 UTC m=+0.207168600 container create 93646214f1310e9ab1ae16335b02918e4de8f82cb8b3ba3eac87b65392e867c3 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-3-deactivate, org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.build-date=20240326) 2024-03-26T21:05:24.515 INFO:teuthology.orchestra.run.smithi063.stdout: 5 239670 10.41 MB/sec execute 438 sec latency 38.191 ms 2024-03-26T21:05:24.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:24 smithi174 ceph-mon[53014]: osdmap e109: 8 total, 7 up, 8 in 2024-03-26T21:05:24.754 INFO:journalctl@ceph.osd.3.smithi063.stdout:Mar 26 21:05:24 smithi063 podman[80329]: 2024-03-26 21:05:24.545891107 +0000 UTC m=+0.595471096 container init 93646214f1310e9ab1ae16335b02918e4de8f82cb8b3ba3eac87b65392e867c3 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-3-deactivate, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.build-date=20240326, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e) 2024-03-26T21:05:24.754 INFO:journalctl@ceph.osd.3.smithi063.stdout:Mar 26 21:05:24 smithi063 podman[80329]: 2024-03-26 21:05:24.553806328 +0000 UTC m=+0.603386304 container start 93646214f1310e9ab1ae16335b02918e4de8f82cb8b3ba3eac87b65392e867c3 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-3-deactivate, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.build-date=20240326, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, ceph=True, org.label-schema.schema-version=1.0) 2024-03-26T21:05:24.754 INFO:journalctl@ceph.osd.3.smithi063.stdout:Mar 26 21:05:24 smithi063 podman[80329]: 2024-03-26 21:05:24.751784192 +0000 UTC m=+0.801364171 container attach 93646214f1310e9ab1ae16335b02918e4de8f82cb8b3ba3eac87b65392e867c3 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-3-deactivate, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-03-26T21:05:25.005 INFO:journalctl@ceph.osd.3.smithi063.stdout:Mar 26 21:05:25 smithi063 podman[80329]: 2024-03-26 21:05:25.003628539 +0000 UTC m=+1.053208519 container died 93646214f1310e9ab1ae16335b02918e4de8f82cb8b3ba3eac87b65392e867c3 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-3-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240326, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, org.label-schema.license=GPLv2, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-03-26T21:05:25.255 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:25 smithi063 ceph-mon[66934]: pgmap v166: 97 pgs: 16 stale+active+clean, 81 active+clean; 347 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 5.0 KiB/s rd, 6.2 MiB/s wr, 556 op/s 2024-03-26T21:05:25.516 INFO:teuthology.orchestra.run.smithi063.stdout: 5 240108 10.42 MB/sec execute 439 sec latency 35.547 ms 2024-03-26T21:05:25.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:25 smithi174 ceph-mon[53014]: pgmap v166: 97 pgs: 16 stale+active+clean, 81 active+clean; 347 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 5.0 KiB/s rd, 6.2 MiB/s wr, 556 op/s 2024-03-26T21:05:26.293 INFO:journalctl@ceph.osd.3.smithi063.stdout:Mar 26 21:05:26 smithi063 podman[80353]: 2024-03-26 21:05:26.274586102 +0000 UTC m=+1.264542511 container remove 93646214f1310e9ab1ae16335b02918e4de8f82cb8b3ba3eac87b65392e867c3 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-3-deactivate, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326) 2024-03-26T21:05:26.517 INFO:teuthology.orchestra.run.smithi063.stdout: 5 240438 10.41 MB/sec execute 440 sec latency 64.153 ms 2024-03-26T21:05:26.551 INFO:journalctl@ceph.osd.3.smithi063.stdout:Mar 26 21:05:26 smithi063 systemd[1]: ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715@osd.3.service: Deactivated successfully. 2024-03-26T21:05:26.551 INFO:journalctl@ceph.osd.3.smithi063.stdout:Mar 26 21:05:26 smithi063 systemd[1]: Stopped Ceph osd.3 for 0ed4dd4c-ebb2-11ee-95d0-87774f69a715. 2024-03-26T21:05:26.551 INFO:journalctl@ceph.osd.3.smithi063.stdout:Mar 26 21:05:26 smithi063 systemd[1]: ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715@osd.3.service: Consumed 4min 57.743s CPU time. 2024-03-26T21:05:26.809 INFO:journalctl@ceph.osd.3.smithi063.stdout:Mar 26 21:05:26 smithi063 systemd[1]: Starting Ceph osd.3 for 0ed4dd4c-ebb2-11ee-95d0-87774f69a715... 2024-03-26T21:05:26.809 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:26 smithi063 ceph-mon[66934]: pgmap v167: 97 pgs: 9 active+undersized+degraded, 11 stale+active+clean, 5 active+undersized, 72 active+clean; 337 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 3.9 MiB/s wr, 481 op/s; 42/1575 objects degraded (2.667%) 2024-03-26T21:05:26.809 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:26 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:05:26.809 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:26 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:05:26.809 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:26 smithi063 ceph-mon[66934]: Health check failed: Degraded data redundancy: 42/1575 objects degraded (2.667%), 9 pgs degraded (PG_DEGRADED) 2024-03-26T21:05:27.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:26 smithi174 ceph-mon[53014]: pgmap v167: 97 pgs: 9 active+undersized+degraded, 11 stale+active+clean, 5 active+undersized, 72 active+clean; 337 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 3.9 MiB/s wr, 481 op/s; 42/1575 objects degraded (2.667%) 2024-03-26T21:05:27.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:26 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:05:27.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:26 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:05:27.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:26 smithi174 ceph-mon[53014]: Health check failed: Degraded data redundancy: 42/1575 objects degraded (2.667%), 9 pgs degraded (PG_DEGRADED) 2024-03-26T21:05:27.061 INFO:journalctl@ceph.osd.3.smithi063.stdout:Mar 26 21:05:26 smithi063 podman[80448]: 2024-03-26 21:05:26.72266922 +0000 UTC m=+0.022582854 image pull a9bdc3eaeb106993bcbbac35d7fe5c7b76c052021676e129f5984aed30aee063 quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d 2024-03-26T21:05:27.061 INFO:journalctl@ceph.osd.3.smithi063.stdout:Mar 26 21:05:26 smithi063 podman[80448]: 2024-03-26 21:05:26.868352918 +0000 UTC m=+0.168266517 container create e7460b116379a0adfc0e12b4176e08c0f6656d377537f0c10a4c87924a1e7f6a (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-3-activate, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240326, GIT_CLEAN=True, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e) 2024-03-26T21:05:27.312 INFO:journalctl@ceph.osd.3.smithi063.stdout:Mar 26 21:05:27 smithi063 podman[80448]: 2024-03-26 21:05:27.222623244 +0000 UTC m=+0.522536853 container init e7460b116379a0adfc0e12b4176e08c0f6656d377537f0c10a4c87924a1e7f6a (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-3-activate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240326, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-03-26T21:05:27.312 INFO:journalctl@ceph.osd.3.smithi063.stdout:Mar 26 21:05:27 smithi063 podman[80448]: 2024-03-26 21:05:27.233120215 +0000 UTC m=+0.533033817 container start e7460b116379a0adfc0e12b4176e08c0f6656d377537f0c10a4c87924a1e7f6a (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-3-activate, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-26T21:05:27.519 INFO:teuthology.orchestra.run.smithi063.stdout: 5 240721 10.41 MB/sec execute 441 sec latency 46.970 ms 2024-03-26T21:05:27.563 INFO:journalctl@ceph.osd.3.smithi063.stdout:Mar 26 21:05:27 smithi063 podman[80448]: 2024-03-26 21:05:27.32251319 +0000 UTC m=+0.622426794 container attach e7460b116379a0adfc0e12b4176e08c0f6656d377537f0c10a4c87924a1e7f6a (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-3-activate, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-03-26T21:05:27.572 INFO:teuthology.orchestra.run.smithi063.stdout:true 2024-03-26T21:05:28.003 INFO:journalctl@ceph.osd.3.smithi063.stdout:Mar 26 21:05:27 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-3-activate[80462]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-03-26T21:05:28.003 INFO:journalctl@ceph.osd.3.smithi063.stdout:Mar 26 21:05:27 smithi063 bash[80448]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-03-26T21:05:28.003 INFO:journalctl@ceph.osd.3.smithi063.stdout:Mar 26 21:05:27 smithi063 bash[80448]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-03-26T21:05:28.003 INFO:journalctl@ceph.osd.3.smithi063.stdout:Mar 26 21:05:27 smithi063 bash[80448]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-03-26T21:05:28.003 INFO:journalctl@ceph.osd.3.smithi063.stdout:Mar 26 21:05:27 smithi063 bash[80448]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-03-26T21:05:28.003 INFO:journalctl@ceph.osd.3.smithi063.stdout:Mar 26 21:05:27 smithi063 bash[80448]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-03-26T21:05:28.003 INFO:journalctl@ceph.osd.3.smithi063.stdout:Mar 26 21:05:27 smithi063 bash[80448]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-03-26T21:05:28.004 INFO:journalctl@ceph.osd.3.smithi063.stdout:Mar 26 21:05:27 smithi063 bash[80448]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-03-26T21:05:28.004 INFO:journalctl@ceph.osd.3.smithi063.stdout:Mar 26 21:05:27 smithi063 bash[80448]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-03-26T21:05:28.004 INFO:journalctl@ceph.osd.3.smithi063.stdout:Mar 26 21:05:27 smithi063 bash[80448]: --> ceph-volume raw activate successful for osd ID: 3 2024-03-26T21:05:28.004 INFO:journalctl@ceph.osd.3.smithi063.stdout:Mar 26 21:05:27 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-3-activate[80462]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-03-26T21:05:28.004 INFO:journalctl@ceph.osd.3.smithi063.stdout:Mar 26 21:05:27 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-3-activate[80462]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-03-26T21:05:28.004 INFO:journalctl@ceph.osd.3.smithi063.stdout:Mar 26 21:05:27 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-3-activate[80462]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-03-26T21:05:28.004 INFO:journalctl@ceph.osd.3.smithi063.stdout:Mar 26 21:05:27 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-3-activate[80462]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-03-26T21:05:28.004 INFO:journalctl@ceph.osd.3.smithi063.stdout:Mar 26 21:05:27 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-3-activate[80462]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-03-26T21:05:28.004 INFO:journalctl@ceph.osd.3.smithi063.stdout:Mar 26 21:05:27 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-3-activate[80462]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-03-26T21:05:28.004 INFO:journalctl@ceph.osd.3.smithi063.stdout:Mar 26 21:05:27 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-3-activate[80462]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-03-26T21:05:28.004 INFO:journalctl@ceph.osd.3.smithi063.stdout:Mar 26 21:05:27 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-3-activate[80462]: --> ceph-volume raw activate successful for osd ID: 3 2024-03-26T21:05:28.004 INFO:journalctl@ceph.osd.3.smithi063.stdout:Mar 26 21:05:27 smithi063 podman[80448]: 2024-03-26 21:05:27.87024778 +0000 UTC m=+1.170161393 container died e7460b116379a0adfc0e12b4176e08c0f6656d377537f0c10a4c87924a1e7f6a (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-3-activate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, org.label-schema.build-date=20240326, io.buildah.version=1.33.5, GIT_CLEAN=True) 2024-03-26T21:05:28.422 INFO:teuthology.orchestra.run.smithi063.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-26T21:05:28.422 INFO:teuthology.orchestra.run.smithi063.stdout:alertmanager.smithi063 smithi063 *:9093,9094 running (6m) 22s ago 15m 24.5M - 0.23.0 ba2b418f427c 6e160c94aac3 2024-03-26T21:05:28.422 INFO:teuthology.orchestra.run.smithi063.stdout:crash.smithi063 smithi063 running (2m) 22s ago 15m 6665k - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 cf1096d84270 2024-03-26T21:05:28.422 INFO:teuthology.orchestra.run.smithi063.stdout:crash.smithi174 smithi174 running (2m) 2m ago 15m 6665k - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 cb5371a92684 2024-03-26T21:05:28.422 INFO:teuthology.orchestra.run.smithi063.stdout:grafana.smithi063 smithi063 *:3000 error 22s ago 15m - - 2024-03-26T21:05:28.422 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foofs.smithi063.tzaznz smithi063 running (9m) 22s ago 9m 37.9M - 17.2.0 e1d6a67b021e 10a989fc7954 2024-03-26T21:05:28.422 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foofs.smithi174.gitdvk smithi174 running (9m) 2m ago 9m 13.4M - 17.2.0 e1d6a67b021e 4339a5d98d1d 2024-03-26T21:05:28.423 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.smithi063.lxhujt smithi063 *:8443,9283,8765 running (5m) 22s ago 17m 508M - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 6d9d10330a90 2024-03-26T21:05:28.423 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.smithi174.sexoin smithi174 *:8443,9283,8765 running (4m) 2m ago 15m 450M - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 d7546233672d 2024-03-26T21:05:28.423 INFO:teuthology.orchestra.run.smithi063.stdout:mon.smithi063 smithi063 running (4m) 22s ago 17m 51.2M 2048M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 c644200344f8 2024-03-26T21:05:28.423 INFO:teuthology.orchestra.run.smithi063.stdout:mon.smithi174 smithi174 running (3m) 2m ago 15m 39.0M 2048M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 1a6d442692e7 2024-03-26T21:05:28.423 INFO:teuthology.orchestra.run.smithi063.stdout:nfs.foo.0.0.smithi063.fgmpwa smithi063 *:2049 running (9m) 22s ago 9m 275M - 4.0 e1d6a67b021e 3d6c0dd9ce51 2024-03-26T21:05:28.423 INFO:teuthology.orchestra.run.smithi063.stdout:nfs.foo.1.0.smithi174.pytptt smithi174 *:2049 running (9m) 2m ago 9m 86.4M - 4.0 e1d6a67b021e f220dbb449a3 2024-03-26T21:05:28.423 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.smithi063 smithi063 *:9100 running (6m) 22s ago 15m 20.1M - 1.3.1 1dbe0e931976 f52f41fd14c0 2024-03-26T21:05:28.423 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.smithi174 smithi174 *:9100 running (5m) 2m ago 14m 17.2M - 1.3.1 1dbe0e931976 0c027f39a4c8 2024-03-26T21:05:28.423 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0 smithi063 running (2m) 22s ago 14m 306M 1936M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 2d4b3fa825cf 2024-03-26T21:05:28.423 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1 smithi063 running (79s) 22s ago 13m 244M 1936M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 d41fbe507197 2024-03-26T21:05:28.423 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2 smithi063 running (32s) 22s ago 13m 15.0M 1936M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 f123eeabd3fa 2024-03-26T21:05:28.423 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3 smithi063 running (12m) 22s ago 12m 460M 1936M 17.2.0 e1d6a67b021e ab0f1a855957 2024-03-26T21:05:28.423 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4 smithi174 running (12m) 2m ago 12m 575M 2704M 17.2.0 e1d6a67b021e a0e877ecc06d 2024-03-26T21:05:28.424 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5 smithi174 running (12m) 2m ago 12m 423M 2704M 17.2.0 e1d6a67b021e 378c2aca7e71 2024-03-26T21:05:28.424 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6 smithi174 running (11m) 2m ago 11m 328M 2704M 17.2.0 e1d6a67b021e ffbb5b7ad3b6 2024-03-26T21:05:28.424 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7 smithi174 running (11m) 2m ago 11m 524M 2704M 17.2.0 e1d6a67b021e c1d57fcabb3f 2024-03-26T21:05:28.424 INFO:teuthology.orchestra.run.smithi063.stdout:prometheus.smithi063 smithi063 *:9095 running (4m) 22s ago 14m 72.0M - 2.33.4 514e6a882f6e 567eef202627 2024-03-26T21:05:28.519 INFO:teuthology.orchestra.run.smithi063.stdout: 5 240925 10.39 MB/sec execute 442 sec latency 43.555 ms 2024-03-26T21:05:28.846 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:28 smithi063 ceph-mon[66934]: pgmap v168: 97 pgs: 27 active+undersized+degraded, 14 active+undersized, 56 active+clean; 335 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 5.8 MiB/s wr, 600 op/s; 201/1617 objects degraded (12.430%) 2024-03-26T21:05:28.847 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:28 smithi063 ceph-mon[66934]: from='client.34308 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:05:28.847 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:28 smithi063 ceph-mon[66934]: from='client.34312 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:05:28.893 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-03-26T21:05:28.893 INFO:teuthology.orchestra.run.smithi063.stdout: "mon": { 2024-03-26T21:05:28.893 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 2 2024-03-26T21:05:28.893 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:05:28.893 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr": { 2024-03-26T21:05:28.893 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 2 2024-03-26T21:05:28.893 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:05:28.893 INFO:teuthology.orchestra.run.smithi063.stdout: "osd": { 2024-03-26T21:05:28.893 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-03-26T21:05:28.893 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 3 2024-03-26T21:05:28.893 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:05:28.893 INFO:teuthology.orchestra.run.smithi063.stdout: "mds": { 2024-03-26T21:05:28.893 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-26T21:05:28.894 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:05:28.894 INFO:teuthology.orchestra.run.smithi063.stdout: "overall": { 2024-03-26T21:05:28.894 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-03-26T21:05:28.894 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 7 2024-03-26T21:05:28.894 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-03-26T21:05:28.894 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-03-26T21:05:29.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:28 smithi174 ceph-mon[53014]: pgmap v168: 97 pgs: 27 active+undersized+degraded, 14 active+undersized, 56 active+clean; 335 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 5.8 MiB/s wr, 600 op/s; 201/1617 objects degraded (12.430%) 2024-03-26T21:05:29.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:28 smithi174 ceph-mon[53014]: from='client.34308 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:05:29.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:28 smithi174 ceph-mon[53014]: from='client.34312 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:05:29.297 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-03-26T21:05:29.298 INFO:teuthology.orchestra.run.smithi063.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d", 2024-03-26T21:05:29.298 INFO:teuthology.orchestra.run.smithi063.stdout: "in_progress": true, 2024-03-26T21:05:29.298 INFO:teuthology.orchestra.run.smithi063.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-26T21:05:29.298 INFO:teuthology.orchestra.run.smithi063.stdout: "services_complete": [ 2024-03-26T21:05:29.298 INFO:teuthology.orchestra.run.smithi063.stdout: "crash", 2024-03-26T21:05:29.298 INFO:teuthology.orchestra.run.smithi063.stdout: "mon", 2024-03-26T21:05:29.298 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr" 2024-03-26T21:05:29.298 INFO:teuthology.orchestra.run.smithi063.stdout: ], 2024-03-26T21:05:29.298 INFO:teuthology.orchestra.run.smithi063.stdout: "progress": "9/23 daemons upgraded", 2024-03-26T21:05:29.298 INFO:teuthology.orchestra.run.smithi063.stdout: "message": "Currently upgrading osd daemons", 2024-03-26T21:05:29.298 INFO:teuthology.orchestra.run.smithi063.stdout: "is_paused": false 2024-03-26T21:05:29.299 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-03-26T21:05:29.519 INFO:teuthology.orchestra.run.smithi063.stdout: 5 241167 10.39 MB/sec execute 443 sec latency 60.714 ms 2024-03-26T21:05:29.793 INFO:teuthology.orchestra.run.smithi063.stdout:HEALTH_WARN 1 failed cephadm daemon(s); 1 osds down; Degraded data redundancy: 201/1617 objects degraded (12.430%), 27 pgs degraded 2024-03-26T21:05:29.794 INFO:teuthology.orchestra.run.smithi063.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-26T21:05:29.794 INFO:teuthology.orchestra.run.smithi063.stdout: daemon grafana.smithi063 on smithi063 is in error state 2024-03-26T21:05:29.794 INFO:teuthology.orchestra.run.smithi063.stdout:[WRN] OSD_DOWN: 1 osds down 2024-03-26T21:05:29.794 INFO:teuthology.orchestra.run.smithi063.stdout: osd.3 (root=default,host=smithi063) is down 2024-03-26T21:05:29.794 INFO:teuthology.orchestra.run.smithi063.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 201/1617 objects degraded (12.430%), 27 pgs degraded 2024-03-26T21:05:29.794 INFO:teuthology.orchestra.run.smithi063.stdout: pg 2.1 is active+undersized+degraded, acting [2,0] 2024-03-26T21:05:29.794 INFO:teuthology.orchestra.run.smithi063.stdout: pg 2.9 is active+undersized+degraded, acting [1,7] 2024-03-26T21:05:29.794 INFO:teuthology.orchestra.run.smithi063.stdout: pg 2.a is active+undersized+degraded, acting [1,7] 2024-03-26T21:05:29.794 INFO:teuthology.orchestra.run.smithi063.stdout: pg 2.d is active+undersized+degraded, acting [1,4] 2024-03-26T21:05:29.794 INFO:teuthology.orchestra.run.smithi063.stdout: pg 2.e is active+undersized+degraded, acting [2,7] 2024-03-26T21:05:29.794 INFO:teuthology.orchestra.run.smithi063.stdout: pg 2.12 is active+undersized+degraded, acting [5,7] 2024-03-26T21:05:29.794 INFO:teuthology.orchestra.run.smithi063.stdout: pg 2.13 is active+undersized+degraded, acting [0,4] 2024-03-26T21:05:29.795 INFO:teuthology.orchestra.run.smithi063.stdout: pg 2.14 is active+undersized+degraded, acting [6,5] 2024-03-26T21:05:29.795 INFO:teuthology.orchestra.run.smithi063.stdout: pg 2.19 is active+undersized+degraded, acting [6,0] 2024-03-26T21:05:29.795 INFO:teuthology.orchestra.run.smithi063.stdout: pg 2.1b is active+undersized+degraded, acting [7,6] 2024-03-26T21:05:29.795 INFO:teuthology.orchestra.run.smithi063.stdout: pg 2.1e is active+undersized+degraded, acting [0,5] 2024-03-26T21:05:29.795 INFO:teuthology.orchestra.run.smithi063.stdout: pg 3.1 is active+undersized+degraded, acting [0,4] 2024-03-26T21:05:29.795 INFO:teuthology.orchestra.run.smithi063.stdout: pg 3.2 is active+undersized+degraded, acting [5,6] 2024-03-26T21:05:29.795 INFO:teuthology.orchestra.run.smithi063.stdout: pg 3.5 is active+undersized+degraded, acting [5,2] 2024-03-26T21:05:29.795 INFO:teuthology.orchestra.run.smithi063.stdout: pg 3.7 is active+undersized+degraded, acting [7,0] 2024-03-26T21:05:29.795 INFO:teuthology.orchestra.run.smithi063.stdout: pg 3.8 is active+undersized+degraded, acting [1,7] 2024-03-26T21:05:29.795 INFO:teuthology.orchestra.run.smithi063.stdout: pg 3.b is active+undersized+degraded, acting [0,4] 2024-03-26T21:05:29.795 INFO:teuthology.orchestra.run.smithi063.stdout: pg 3.c is active+undersized+degraded, acting [5,6] 2024-03-26T21:05:29.795 INFO:teuthology.orchestra.run.smithi063.stdout: pg 3.12 is active+undersized+degraded, acting [0,7] 2024-03-26T21:05:29.795 INFO:teuthology.orchestra.run.smithi063.stdout: pg 3.15 is active+undersized+degraded, acting [7,4] 2024-03-26T21:05:29.795 INFO:teuthology.orchestra.run.smithi063.stdout: pg 3.17 is active+undersized+degraded, acting [0,5] 2024-03-26T21:05:29.795 INFO:teuthology.orchestra.run.smithi063.stdout: pg 3.18 is active+undersized+degraded, acting [0,1] 2024-03-26T21:05:29.795 INFO:teuthology.orchestra.run.smithi063.stdout: pg 3.19 is active+undersized+degraded, acting [1,4] 2024-03-26T21:05:29.796 INFO:teuthology.orchestra.run.smithi063.stdout: pg 3.1e is active+undersized+degraded, acting [6,2] 2024-03-26T21:05:29.796 INFO:teuthology.orchestra.run.smithi063.stdout: pg 4.0 is active+undersized+degraded, acting [7,0] 2024-03-26T21:05:29.796 INFO:teuthology.orchestra.run.smithi063.stdout: pg 4.14 is active+undersized+degraded, acting [2,7] 2024-03-26T21:05:29.796 INFO:teuthology.orchestra.run.smithi063.stdout: pg 4.19 is active+undersized+degraded, acting [2,0] 2024-03-26T21:05:29.898 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:29 smithi063 ceph-mon[66934]: from='client.34316 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:05:29.898 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:29 smithi063 ceph-mon[66934]: from='client.? 172.21.15.63:0/2741661574' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:05:29.898 INFO:journalctl@ceph.osd.3.smithi063.stdout:Mar 26 21:05:29 smithi063 podman[80652]: 2024-03-26 21:05:29.709490934 +0000 UTC m=+1.833451584 container remove e7460b116379a0adfc0e12b4176e08c0f6656d377537f0c10a4c87924a1e7f6a (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-3-activate, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-03-26T21:05:30.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:29 smithi174 ceph-mon[53014]: from='client.34316 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:05:30.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:29 smithi174 ceph-mon[53014]: from='client.? 172.21.15.63:0/2741661574' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:05:30.153 INFO:journalctl@ceph.osd.3.smithi063.stdout:Mar 26 21:05:29 smithi063 podman[80804]: 2024-03-26 21:05:29.851799111 +0000 UTC m=+0.022319559 image pull a9bdc3eaeb106993bcbbac35d7fe5c7b76c052021676e129f5984aed30aee063 quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d 2024-03-26T21:05:30.405 INFO:journalctl@ceph.osd.3.smithi063.stdout:Mar 26 21:05:30 smithi063 podman[80804]: 2024-03-26 21:05:30.201442082 +0000 UTC m=+0.371962507 container create 953816eafe87e6dacd376c5e8ed51d8307d0b077830d4573c89303a7f2f59183 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-3, org.label-schema.build-date=20240326, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5) 2024-03-26T21:05:30.519 INFO:teuthology.orchestra.run.smithi063.stdout: 5 241386 10.38 MB/sec execute 444 sec latency 59.677 ms 2024-03-26T21:05:30.676 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:30 smithi063 ceph-mon[66934]: from='client.34324 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:05:30.676 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:30 smithi063 ceph-mon[66934]: pgmap v169: 97 pgs: 27 active+undersized+degraded, 14 active+undersized, 56 active+clean; 335 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 4.1 KiB/s rd, 4.9 MiB/s wr, 418 op/s; 201/1617 objects degraded (12.430%) 2024-03-26T21:05:30.676 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:30 smithi063 ceph-mon[66934]: from='client.? 172.21.15.63:0/571937270' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-26T21:05:30.941 INFO:journalctl@ceph.osd.3.smithi063.stdout:Mar 26 21:05:30 smithi063 podman[80804]: 2024-03-26 21:05:30.72775112 +0000 UTC m=+0.898271562 container init 953816eafe87e6dacd376c5e8ed51d8307d0b077830d4573c89303a7f2f59183 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-3, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-26T21:05:30.941 INFO:journalctl@ceph.osd.3.smithi063.stdout:Mar 26 21:05:30 smithi063 podman[80804]: 2024-03-26 21:05:30.735706715 +0000 UTC m=+0.906227143 container start 953816eafe87e6dacd376c5e8ed51d8307d0b077830d4573c89303a7f2f59183 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-3, io.buildah.version=1.33.5, ceph=True, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-03-26T21:05:30.941 INFO:journalctl@ceph.osd.3.smithi063.stdout:Mar 26 21:05:30 smithi063 ceph-osd[80822]: -- 172.21.15.63:0/3844646262 <== mon.0 v2:172.21.15.63:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x562f06a33680 con 0x562f069fd000 2024-03-26T21:05:30.941 INFO:journalctl@ceph.osd.3.smithi063.stdout:Mar 26 21:05:30 smithi063 bash[80804]: 953816eafe87e6dacd376c5e8ed51d8307d0b077830d4573c89303a7f2f59183 2024-03-26T21:05:30.941 INFO:journalctl@ceph.osd.3.smithi063.stdout:Mar 26 21:05:30 smithi063 systemd[1]: Started Ceph osd.3 for 0ed4dd4c-ebb2-11ee-95d0-87774f69a715. 2024-03-26T21:05:31.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:30 smithi174 ceph-mon[53014]: from='client.34324 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:05:31.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:30 smithi174 ceph-mon[53014]: pgmap v169: 97 pgs: 27 active+undersized+degraded, 14 active+undersized, 56 active+clean; 335 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 4.1 KiB/s rd, 4.9 MiB/s wr, 418 op/s; 201/1617 objects degraded (12.430%) 2024-03-26T21:05:31.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:30 smithi174 ceph-mon[53014]: from='client.? 172.21.15.63:0/571937270' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-26T21:05:31.478 INFO:journalctl@ceph.osd.3.smithi063.stdout:Mar 26 21:05:31 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-3[80818]: 2024-03-26T21:05:31.311+0000 7f3c66d997c0 -1 Falling back to public interface 2024-03-26T21:05:31.519 INFO:teuthology.orchestra.run.smithi063.stdout: 5 241677 10.37 MB/sec execute 445 sec latency 52.659 ms 2024-03-26T21:05:31.987 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:31 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:05:31.987 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:31 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:05:31.987 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:31 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:05:32.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:31 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:05:32.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:31 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:05:32.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:31 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:05:32.520 INFO:teuthology.orchestra.run.smithi063.stdout: 5 242062 10.38 MB/sec execute 446 sec latency 54.584 ms 2024-03-26T21:05:33.023 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:32 smithi063 ceph-mon[66934]: pgmap v170: 97 pgs: 27 active+undersized+degraded, 14 active+undersized, 56 active+clean; 333 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 3.4 KiB/s rd, 2.4 MiB/s wr, 250 op/s; 205/1632 objects degraded (12.561%) 2024-03-26T21:05:33.023 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:32 smithi063 ceph-mon[66934]: Health check update: Degraded data redundancy: 205/1632 objects degraded (12.561%), 27 pgs degraded (PG_DEGRADED) 2024-03-26T21:05:33.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:32 smithi174 ceph-mon[53014]: pgmap v170: 97 pgs: 27 active+undersized+degraded, 14 active+undersized, 56 active+clean; 333 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 3.4 KiB/s rd, 2.4 MiB/s wr, 250 op/s; 205/1632 objects degraded (12.561%) 2024-03-26T21:05:33.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:32 smithi174 ceph-mon[53014]: Health check update: Degraded data redundancy: 205/1632 objects degraded (12.561%), 27 pgs degraded (PG_DEGRADED) 2024-03-26T21:05:33.520 INFO:teuthology.orchestra.run.smithi063.stdout: 5 242411 10.37 MB/sec execute 447 sec latency 47.530 ms 2024-03-26T21:05:34.520 INFO:teuthology.orchestra.run.smithi063.stdout: 5 242813 10.36 MB/sec execute 448 sec latency 60.882 ms 2024-03-26T21:05:35.046 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:34 smithi063 ceph-mon[66934]: pgmap v171: 97 pgs: 27 active+undersized+degraded, 14 active+undersized, 56 active+clean; 337 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 6.1 KiB/s rd, 3.9 MiB/s wr, 366 op/s; 201/1566 objects degraded (12.835%) 2024-03-26T21:05:35.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:34 smithi174 ceph-mon[53014]: pgmap v171: 97 pgs: 27 active+undersized+degraded, 14 active+undersized, 56 active+clean; 337 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 6.1 KiB/s rd, 3.9 MiB/s wr, 366 op/s; 201/1566 objects degraded (12.835%) 2024-03-26T21:05:35.520 INFO:teuthology.orchestra.run.smithi063.stdout: 5 243245 10.37 MB/sec execute 449 sec latency 43.026 ms 2024-03-26T21:05:36.520 INFO:teuthology.orchestra.run.smithi063.stdout: 5 243701 10.38 MB/sec execute 450 sec latency 39.725 ms 2024-03-26T21:05:37.102 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:37 smithi063 ceph-mon[66934]: pgmap v172: 97 pgs: 27 active+undersized+degraded, 14 active+undersized, 56 active+clean; 342 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 5.4 KiB/s rd, 4.0 MiB/s wr, 413 op/s; 204/1590 objects degraded (12.830%) 2024-03-26T21:05:37.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:37 smithi174 ceph-mon[53014]: pgmap v172: 97 pgs: 27 active+undersized+degraded, 14 active+undersized, 56 active+clean; 342 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 5.4 KiB/s rd, 4.0 MiB/s wr, 413 op/s; 204/1590 objects degraded (12.830%) 2024-03-26T21:05:37.520 INFO:teuthology.orchestra.run.smithi063.stdout: 5 244144 10.38 MB/sec execute 451 sec latency 50.681 ms 2024-03-26T21:05:38.126 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:38 smithi063 ceph-mon[66934]: Health check update: Degraded data redundancy: 212/1656 objects degraded (12.802%), 27 pgs degraded (PG_DEGRADED) 2024-03-26T21:05:38.520 INFO:teuthology.orchestra.run.smithi063.stdout: 5 244455 10.37 MB/sec execute 452 sec latency 44.336 ms 2024-03-26T21:05:38.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:38 smithi174 ceph-mon[53014]: Health check update: Degraded data redundancy: 212/1656 objects degraded (12.802%), 27 pgs degraded (PG_DEGRADED) 2024-03-26T21:05:39.138 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:39 smithi063 ceph-mon[66934]: pgmap v173: 97 pgs: 27 active+undersized+degraded, 14 active+undersized, 56 active+clean; 346 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.5 MiB/s wr, 526 op/s; 212/1656 objects degraded (12.802%) 2024-03-26T21:05:39.138 INFO:journalctl@ceph.osd.3.smithi063.stdout:Mar 26 21:05:38 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-3[80818]: 2024-03-26T21:05:38.994+0000 7f3c66d997c0 -1 osd.3 0 read_superblock omap replica is missing. 2024-03-26T21:05:39.520 INFO:teuthology.orchestra.run.smithi063.stdout: 5 244782 10.37 MB/sec execute 453 sec latency 43.311 ms 2024-03-26T21:05:39.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:39 smithi174 ceph-mon[53014]: pgmap v173: 97 pgs: 27 active+undersized+degraded, 14 active+undersized, 56 active+clean; 346 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.5 MiB/s wr, 526 op/s; 212/1656 objects degraded (12.802%) 2024-03-26T21:05:40.520 INFO:teuthology.orchestra.run.smithi063.stdout: 5 245070 10.37 MB/sec execute 454 sec latency 59.311 ms 2024-03-26T21:05:40.672 INFO:journalctl@ceph.osd.3.smithi063.stdout:Mar 26 21:05:40 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-3[80818]: 2024-03-26T21:05:40.583+0000 7f3c66d997c0 -1 osd.3 107 log_to_monitors true 2024-03-26T21:05:41.449 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:41 smithi063 ceph-mon[66934]: pgmap v174: 97 pgs: 27 active+undersized+degraded, 14 active+undersized, 56 active+clean; 346 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.3 MiB/s wr, 446 op/s; 212/1656 objects degraded (12.802%) 2024-03-26T21:05:41.449 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:41 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:05:41.449 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:41 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:05:41.449 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:41 smithi063 ceph-mon[66934]: from='osd.3 [v2:172.21.15.63:6826/2652577981,v1:172.21.15.63:6827/2652577981]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-03-26T21:05:41.449 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:41 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:05:41.449 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:41 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:05:41.520 INFO:teuthology.orchestra.run.smithi063.stdout: 5 245319 10.36 MB/sec execute 455 sec latency 90.762 ms 2024-03-26T21:05:41.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:41 smithi174 ceph-mon[53014]: pgmap v174: 97 pgs: 27 active+undersized+degraded, 14 active+undersized, 56 active+clean; 346 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.3 MiB/s wr, 446 op/s; 212/1656 objects degraded (12.802%) 2024-03-26T21:05:41.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:41 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:05:41.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:41 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:05:41.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:41 smithi174 ceph-mon[53014]: from='osd.3 [v2:172.21.15.63:6826/2652577981,v1:172.21.15.63:6827/2652577981]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-03-26T21:05:41.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:41 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:05:41.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:41 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:05:41.951 INFO:journalctl@ceph.osd.3.smithi063.stdout:Mar 26 21:05:41 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-3[80818]: 2024-03-26T21:05:41.740+0000 7f3c541e9700 -1 osd.3 107 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-26T21:05:42.201 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:42 smithi063 ceph-mon[66934]: from='osd.3 [v2:172.21.15.63:6826/2652577981,v1:172.21.15.63:6827/2652577981]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-03-26T21:05:42.202 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:42 smithi063 ceph-mon[66934]: osdmap e110: 8 total, 7 up, 8 in 2024-03-26T21:05:42.202 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:42 smithi063 ceph-mon[66934]: from='osd.3 [v2:172.21.15.63:6826/2652577981,v1:172.21.15.63:6827/2652577981]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi063", "root=default"]}]: dispatch 2024-03-26T21:05:42.202 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:42 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:05:42.202 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:42 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:05:42.202 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:42 smithi063 ceph-mon[66934]: from='osd.3 [v2:172.21.15.63:6826/2652577981,v1:172.21.15.63:6827/2652577981]' entity='osd.3' 2024-03-26T21:05:42.520 INFO:teuthology.orchestra.run.smithi063.stdout: 5 245630 10.37 MB/sec execute 456 sec latency 79.835 ms 2024-03-26T21:05:42.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:42 smithi174 ceph-mon[53014]: from='osd.3 [v2:172.21.15.63:6826/2652577981,v1:172.21.15.63:6827/2652577981]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-03-26T21:05:42.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:42 smithi174 ceph-mon[53014]: osdmap e110: 8 total, 7 up, 8 in 2024-03-26T21:05:42.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:42 smithi174 ceph-mon[53014]: from='osd.3 [v2:172.21.15.63:6826/2652577981,v1:172.21.15.63:6827/2652577981]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi063", "root=default"]}]: dispatch 2024-03-26T21:05:42.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:42 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:05:42.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:42 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:05:42.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:42 smithi174 ceph-mon[53014]: from='osd.3 [v2:172.21.15.63:6826/2652577981,v1:172.21.15.63:6827/2652577981]' entity='osd.3' 2024-03-26T21:05:43.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:43 smithi063 ceph-mon[66934]: pgmap v176: 97 pgs: 27 active+undersized+degraded, 14 active+undersized, 56 active+clean; 340 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.4 MiB/s wr, 595 op/s; 206/1611 objects degraded (12.787%) 2024-03-26T21:05:43.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:43 smithi063 ceph-mon[66934]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-26T21:05:43.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:43 smithi063 ceph-mon[66934]: osd.3 [v2:172.21.15.63:6826/2652577981,v1:172.21.15.63:6827/2652577981] boot 2024-03-26T21:05:43.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:43 smithi063 ceph-mon[66934]: osdmap e111: 8 total, 8 up, 8 in 2024-03-26T21:05:43.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:43 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-26T21:05:43.520 INFO:teuthology.orchestra.run.smithi063.stdout: 5 245665 10.35 MB/sec execute 457 sec latency 812.630 ms 2024-03-26T21:05:43.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:43 smithi174 ceph-mon[53014]: pgmap v176: 97 pgs: 27 active+undersized+degraded, 14 active+undersized, 56 active+clean; 340 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.4 MiB/s wr, 595 op/s; 206/1611 objects degraded (12.787%) 2024-03-26T21:05:43.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:43 smithi174 ceph-mon[53014]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-26T21:05:43.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:43 smithi174 ceph-mon[53014]: osd.3 [v2:172.21.15.63:6826/2652577981,v1:172.21.15.63:6827/2652577981] boot 2024-03-26T21:05:43.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:43 smithi174 ceph-mon[53014]: osdmap e111: 8 total, 8 up, 8 in 2024-03-26T21:05:43.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:43 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-26T21:05:44.520 INFO:teuthology.orchestra.run.smithi063.stdout: 5 245665 10.32 MB/sec execute 458 sec latency 1812.657 ms 2024-03-26T21:05:44.937 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:44 smithi063 ceph-mon[66934]: pgmap v178: 97 pgs: 27 active+undersized+degraded, 14 active+undersized, 56 active+clean; 338 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.6 KiB/s rd, 5.8 MiB/s wr, 509 op/s; 187/1572 objects degraded (11.896%) 2024-03-26T21:05:44.937 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:44 smithi063 ceph-mon[66934]: osdmap e112: 8 total, 8 up, 8 in 2024-03-26T21:05:44.937 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:44 smithi063 ceph-mon[66934]: Health check update: Degraded data redundancy: 187/1572 objects degraded (11.896%), 27 pgs degraded (PG_DEGRADED) 2024-03-26T21:05:45.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:44 smithi174 ceph-mon[53014]: pgmap v178: 97 pgs: 27 active+undersized+degraded, 14 active+undersized, 56 active+clean; 338 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.6 KiB/s rd, 5.8 MiB/s wr, 509 op/s; 187/1572 objects degraded (11.896%) 2024-03-26T21:05:45.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:44 smithi174 ceph-mon[53014]: osdmap e112: 8 total, 8 up, 8 in 2024-03-26T21:05:45.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:44 smithi174 ceph-mon[53014]: Health check update: Degraded data redundancy: 187/1572 objects degraded (11.896%), 27 pgs degraded (PG_DEGRADED) 2024-03-26T21:05:45.521 INFO:teuthology.orchestra.run.smithi063.stdout: 5 245828 10.30 MB/sec execute 459 sec latency 2168.931 ms 2024-03-26T21:05:45.739 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:45 smithi063 ceph-mon[66934]: osdmap e113: 8 total, 8 up, 8 in 2024-03-26T21:05:46.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:45 smithi174 ceph-mon[53014]: osdmap e113: 8 total, 8 up, 8 in 2024-03-26T21:05:46.520 INFO:teuthology.orchestra.run.smithi063.stdout: 5 246026 10.28 MB/sec execute 460 sec latency 745.607 ms 2024-03-26T21:05:46.779 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:46 smithi063 ceph-mon[66934]: pgmap v181: 97 pgs: 1 active+undersized+remapped, 3 remapped+peering, 20 active+undersized+degraded, 11 active+undersized, 62 active+clean; 337 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.2 MiB/s wr, 376 op/s; 146/1587 objects degraded (9.200%); 1.8 MiB/s, 4 objects/s recovering 2024-03-26T21:05:46.780 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:46 smithi063 ceph-mon[66934]: osdmap e114: 8 total, 8 up, 8 in 2024-03-26T21:05:47.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:46 smithi174 ceph-mon[53014]: pgmap v181: 97 pgs: 1 active+undersized+remapped, 3 remapped+peering, 20 active+undersized+degraded, 11 active+undersized, 62 active+clean; 337 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.2 MiB/s wr, 376 op/s; 146/1587 objects degraded (9.200%); 1.8 MiB/s, 4 objects/s recovering 2024-03-26T21:05:47.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:46 smithi174 ceph-mon[53014]: osdmap e114: 8 total, 8 up, 8 in 2024-03-26T21:05:47.520 INFO:teuthology.orchestra.run.smithi063.stdout: 5 246250 10.27 MB/sec execute 461 sec latency 986.905 ms 2024-03-26T21:05:47.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:47 smithi063 ceph-mon[66934]: osdmap e115: 8 total, 8 up, 8 in 2024-03-26T21:05:47.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:47 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:05:47.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:47 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:05:47.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:47 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:05:47.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:47 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T21:05:47.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:47 smithi063 ceph-mon[66934]: pgmap v184: 97 pgs: 3 peering, 82 active+clean, 2 active+undersized, 2 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 4 active+undersized+degraded, 3 active+undersized+remapped; 333 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 2.5 MiB/s wr, 403 op/s; 43/1572 objects degraded (2.735%); 2.4 MiB/s, 60 objects/s recovering 2024-03-26T21:05:47.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:47 smithi063 ceph-mon[66934]: pgmap v185: 97 pgs: 3 peering, 82 active+clean, 2 active+undersized, 2 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 4 active+undersized+degraded, 3 active+undersized+remapped; 333 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 2.7 MiB/s wr, 431 op/s; 43/1572 objects degraded (2.735%); 2.6 MiB/s, 65 objects/s recovering 2024-03-26T21:05:47.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:47 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:05:47.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:47 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:05:47.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:47 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:05:47.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:47 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:05:47.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:47 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:05:47.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:47 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:05:47.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:47 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-03-26T21:05:47.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:47 smithi063 ceph-mon[66934]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-03-26T21:05:47.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:47 smithi063 ceph-mon[66934]: Upgrade: unsafe to stop osd(s) at this time (5 PGs are or would become offline) 2024-03-26T21:05:47.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:47 smithi063 ceph-mon[66934]: osdmap e116: 8 total, 8 up, 8 in 2024-03-26T21:05:48.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:47 smithi174 ceph-mon[53014]: osdmap e115: 8 total, 8 up, 8 in 2024-03-26T21:05:48.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:47 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:05:48.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:47 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:05:48.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:47 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:05:48.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:47 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T21:05:48.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:47 smithi174 ceph-mon[53014]: pgmap v184: 97 pgs: 3 peering, 82 active+clean, 2 active+undersized, 2 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 4 active+undersized+degraded, 3 active+undersized+remapped; 333 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 2.5 MiB/s wr, 403 op/s; 43/1572 objects degraded (2.735%); 2.4 MiB/s, 60 objects/s recovering 2024-03-26T21:05:48.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:47 smithi174 ceph-mon[53014]: pgmap v185: 97 pgs: 3 peering, 82 active+clean, 2 active+undersized, 2 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 4 active+undersized+degraded, 3 active+undersized+remapped; 333 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 2.7 MiB/s wr, 431 op/s; 43/1572 objects degraded (2.735%); 2.6 MiB/s, 65 objects/s recovering 2024-03-26T21:05:48.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:47 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:05:48.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:47 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:05:48.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:47 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:05:48.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:47 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:05:48.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:47 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:05:48.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:47 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:05:48.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:47 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-03-26T21:05:48.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:47 smithi174 ceph-mon[53014]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-03-26T21:05:48.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:47 smithi174 ceph-mon[53014]: Upgrade: unsafe to stop osd(s) at this time (5 PGs are or would become offline) 2024-03-26T21:05:48.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:47 smithi174 ceph-mon[53014]: osdmap e116: 8 total, 8 up, 8 in 2024-03-26T21:05:48.521 INFO:teuthology.orchestra.run.smithi063.stdout: 5 246499 10.26 MB/sec execute 462 sec latency 929.682 ms 2024-03-26T21:05:49.520 INFO:teuthology.orchestra.run.smithi063.stdout: 5 246749 10.25 MB/sec execute 463 sec latency 345.177 ms 2024-03-26T21:05:49.790 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:49 smithi063 ceph-mon[66934]: osdmap e117: 8 total, 8 up, 8 in 2024-03-26T21:05:49.791 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:49 smithi063 ceph-mon[66934]: pgmap v188: 97 pgs: 5 peering, 86 active+clean, 2 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+remapped; 333 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 1.7 MiB/s wr, 188 op/s; 5/1560 objects degraded (0.321%); 790 KiB/s, 80 objects/s recovering 2024-03-26T21:05:49.791 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:49 smithi063 ceph-mon[66934]: Health check update: Degraded data redundancy: 43/1572 objects degraded (2.735%), 5 pgs degraded (PG_DEGRADED) 2024-03-26T21:05:50.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:49 smithi174 ceph-mon[53014]: osdmap e117: 8 total, 8 up, 8 in 2024-03-26T21:05:50.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:49 smithi174 ceph-mon[53014]: pgmap v188: 97 pgs: 5 peering, 86 active+clean, 2 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+remapped; 333 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 1.7 MiB/s wr, 188 op/s; 5/1560 objects degraded (0.321%); 790 KiB/s, 80 objects/s recovering 2024-03-26T21:05:50.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:49 smithi174 ceph-mon[53014]: Health check update: Degraded data redundancy: 43/1572 objects degraded (2.735%), 5 pgs degraded (PG_DEGRADED) 2024-03-26T21:05:50.520 INFO:teuthology.orchestra.run.smithi063.stdout: 5 247045 10.25 MB/sec execute 464 sec latency 83.528 ms 2024-03-26T21:05:51.521 INFO:teuthology.orchestra.run.smithi063.stdout: 5 247476 10.26 MB/sec execute 465 sec latency 55.237 ms 2024-03-26T21:05:52.097 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:52 smithi063 ceph-mon[66934]: pgmap v189: 97 pgs: 5 peering, 89 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+undersized+remapped; 332 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 503 KiB/s wr, 42 op/s; 5/1542 objects degraded (0.324%); 2.2 MiB/s, 16 objects/s recovering 2024-03-26T21:05:52.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:52 smithi174 ceph-mon[53014]: pgmap v189: 97 pgs: 5 peering, 89 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+undersized+remapped; 332 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 503 KiB/s wr, 42 op/s; 5/1542 objects degraded (0.324%); 2.2 MiB/s, 16 objects/s recovering 2024-03-26T21:05:52.520 INFO:teuthology.orchestra.run.smithi063.stdout: 5 247812 10.25 MB/sec execute 466 sec latency 57.991 ms 2024-03-26T21:05:53.122 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:53 smithi063 ceph-mon[66934]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 5/1542 objects degraded (0.324%), 1 pg degraded) 2024-03-26T21:05:53.521 INFO:teuthology.orchestra.run.smithi063.stdout: 5 248112 10.26 MB/sec execute 467 sec latency 51.619 ms 2024-03-26T21:05:53.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:53 smithi174 ceph-mon[53014]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 5/1542 objects degraded (0.324%), 1 pg degraded) 2024-03-26T21:05:54.162 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:54 smithi063 ceph-mon[66934]: pgmap v190: 97 pgs: 97 active+clean; 337 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.8 KiB/s rd, 3.7 MiB/s wr, 376 op/s; 1.5 MiB/s, 13 objects/s recovering 2024-03-26T21:05:54.520 INFO:teuthology.orchestra.run.smithi063.stdout: 5 248415 10.26 MB/sec execute 468 sec latency 46.720 ms 2024-03-26T21:05:54.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:54 smithi174 ceph-mon[53014]: pgmap v190: 97 pgs: 97 active+clean; 337 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.8 KiB/s rd, 3.7 MiB/s wr, 376 op/s; 1.5 MiB/s, 13 objects/s recovering 2024-03-26T21:05:55.521 INFO:teuthology.orchestra.run.smithi063.stdout: 5 248689 10.25 MB/sec execute 469 sec latency 77.771 ms 2024-03-26T21:05:56.244 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:56 smithi063 ceph-mon[66934]: pgmap v191: 97 pgs: 97 active+clean; 334 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3.6 KiB/s rd, 2.9 MiB/s wr, 289 op/s; 1.2 MiB/s, 9 objects/s recovering 2024-03-26T21:05:56.244 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:56 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:05:56.244 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:56 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:05:56.521 INFO:teuthology.orchestra.run.smithi063.stdout: 5 248942 10.24 MB/sec execute 470 sec latency 59.129 ms 2024-03-26T21:05:56.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:56 smithi174 ceph-mon[53014]: pgmap v191: 97 pgs: 97 active+clean; 334 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3.6 KiB/s rd, 2.9 MiB/s wr, 289 op/s; 1.2 MiB/s, 9 objects/s recovering 2024-03-26T21:05:56.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:56 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:05:56.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:56 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:05:57.521 INFO:teuthology.orchestra.run.smithi063.stdout: 5 249158 10.23 MB/sec execute 471 sec latency 63.703 ms 2024-03-26T21:05:58.282 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:05:58 smithi063 ceph-mon[66934]: pgmap v192: 97 pgs: 97 active+clean; 338 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.4 MiB/s wr, 459 op/s; 846 KiB/s, 1 objects/s recovering 2024-03-26T21:05:58.350 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:05:58 smithi174 ceph-mon[53014]: pgmap v192: 97 pgs: 97 active+clean; 338 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.4 MiB/s wr, 459 op/s; 846 KiB/s, 1 objects/s recovering 2024-03-26T21:05:58.521 INFO:teuthology.orchestra.run.smithi063.stdout: 5 249471 10.23 MB/sec execute 472 sec latency 54.772 ms 2024-03-26T21:05:59.521 INFO:teuthology.orchestra.run.smithi063.stdout: 5 249813 10.21 MB/sec execute 473 sec latency 56.426 ms 2024-03-26T21:06:00.314 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:00 smithi063 ceph-mon[66934]: pgmap v193: 97 pgs: 97 active+clean; 340 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.6 MiB/s wr, 431 op/s; 764 KiB/s, 1 objects/s recovering 2024-03-26T21:06:00.414 INFO:teuthology.orchestra.run.smithi063.stdout:true 2024-03-26T21:06:00.521 INFO:teuthology.orchestra.run.smithi063.stdout: 5 250104 10.20 MB/sec execute 474 sec latency 42.072 ms 2024-03-26T21:06:00.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:00 smithi174 ceph-mon[53014]: pgmap v193: 97 pgs: 97 active+clean; 340 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.6 MiB/s wr, 431 op/s; 764 KiB/s, 1 objects/s recovering 2024-03-26T21:06:01.517 INFO:teuthology.orchestra.run.smithi063.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-26T21:06:01.517 INFO:teuthology.orchestra.run.smithi063.stdout:alertmanager.smithi063 smithi063 *:9093,9094 running (7m) 21s ago 16m 24.5M - 0.23.0 ba2b418f427c 6e160c94aac3 2024-03-26T21:06:01.517 INFO:teuthology.orchestra.run.smithi063.stdout:crash.smithi063 smithi063 running (3m) 21s ago 16m 6665k - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 cf1096d84270 2024-03-26T21:06:01.517 INFO:teuthology.orchestra.run.smithi063.stdout:crash.smithi174 smithi174 running (3m) 3m ago 15m 6665k - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 cb5371a92684 2024-03-26T21:06:01.517 INFO:teuthology.orchestra.run.smithi063.stdout:grafana.smithi063 smithi063 *:3000 error 21s ago 16m - - 2024-03-26T21:06:01.517 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foofs.smithi063.tzaznz smithi063 running (10m) 21s ago 10m 37.8M - 17.2.0 e1d6a67b021e 10a989fc7954 2024-03-26T21:06:01.517 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foofs.smithi174.gitdvk smithi174 running (10m) 3m ago 10m 13.4M - 17.2.0 e1d6a67b021e 4339a5d98d1d 2024-03-26T21:06:01.517 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.smithi063.lxhujt smithi063 *:8443,9283,8765 running (6m) 21s ago 17m 509M - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 6d9d10330a90 2024-03-26T21:06:01.517 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.smithi174.sexoin smithi174 *:8443,9283,8765 running (5m) 3m ago 15m 450M - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 d7546233672d 2024-03-26T21:06:01.517 INFO:teuthology.orchestra.run.smithi063.stdout:mon.smithi063 smithi063 running (4m) 21s ago 17m 52.3M 2048M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 c644200344f8 2024-03-26T21:06:01.517 INFO:teuthology.orchestra.run.smithi063.stdout:mon.smithi174 smithi174 running (4m) 3m ago 15m 39.0M 2048M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 1a6d442692e7 2024-03-26T21:06:01.518 INFO:teuthology.orchestra.run.smithi063.stdout:nfs.foo.0.0.smithi063.fgmpwa smithi063 *:2049 running (10m) 21s ago 10m 275M - 4.0 e1d6a67b021e 3d6c0dd9ce51 2024-03-26T21:06:01.518 INFO:teuthology.orchestra.run.smithi063.stdout:nfs.foo.1.0.smithi174.pytptt smithi174 *:2049 running (10m) 3m ago 10m 86.4M - 4.0 e1d6a67b021e f220dbb449a3 2024-03-26T21:06:01.518 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.smithi063 smithi063 *:9100 running (6m) 21s ago 16m 20.7M - 1.3.1 1dbe0e931976 f52f41fd14c0 2024-03-26T21:06:01.518 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.smithi174 smithi174 *:9100 running (6m) 3m ago 15m 17.2M - 1.3.1 1dbe0e931976 0c027f39a4c8 2024-03-26T21:06:01.518 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0 smithi063 running (2m) 21s ago 14m 339M 1936M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 2d4b3fa825cf 2024-03-26T21:06:01.518 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1 smithi063 running (112s) 21s ago 14m 282M 1936M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 d41fbe507197 2024-03-26T21:06:01.518 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2 smithi063 running (65s) 21s ago 13m 200M 1936M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 f123eeabd3fa 2024-03-26T21:06:01.518 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3 smithi063 running (31s) 21s ago 13m 15.3M 1936M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 953816eafe87 2024-03-26T21:06:01.518 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4 smithi174 running (13m) 3m ago 13m 575M 2704M 17.2.0 e1d6a67b021e a0e877ecc06d 2024-03-26T21:06:01.518 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5 smithi174 running (12m) 3m ago 12m 423M 2704M 17.2.0 e1d6a67b021e 378c2aca7e71 2024-03-26T21:06:01.518 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6 smithi174 running (12m) 3m ago 12m 328M 2704M 17.2.0 e1d6a67b021e ffbb5b7ad3b6 2024-03-26T21:06:01.518 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7 smithi174 running (11m) 3m ago 11m 524M 2704M 17.2.0 e1d6a67b021e c1d57fcabb3f 2024-03-26T21:06:01.518 INFO:teuthology.orchestra.run.smithi063.stdout:prometheus.smithi063 smithi063 *:9095 running (5m) 21s ago 15m 76.0M - 2.33.4 514e6a882f6e 567eef202627 2024-03-26T21:06:01.521 INFO:teuthology.orchestra.run.smithi063.stdout: 5 250324 10.19 MB/sec execute 475 sec latency 52.582 ms 2024-03-26T21:06:01.888 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:01 smithi063 ceph-mon[66934]: from='client.44253 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:06:01.888 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:01 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:06:01.888 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:01 smithi063 ceph-mon[66934]: from='client.34340 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:06:01.888 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:01 smithi063 ceph-mon[66934]: pgmap v194: 97 pgs: 97 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 4.2 MiB/s wr, 442 op/s; 650 KiB/s, 0 objects/s recovering 2024-03-26T21:06:02.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:01 smithi174 ceph-mon[53014]: from='client.44253 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:06:02.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:01 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:06:02.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:01 smithi174 ceph-mon[53014]: from='client.34340 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:06:02.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:01 smithi174 ceph-mon[53014]: pgmap v194: 97 pgs: 97 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 4.2 MiB/s wr, 442 op/s; 650 KiB/s, 0 objects/s recovering 2024-03-26T21:06:02.184 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-03-26T21:06:02.184 INFO:teuthology.orchestra.run.smithi063.stdout: "mon": { 2024-03-26T21:06:02.184 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 2 2024-03-26T21:06:02.184 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:06:02.184 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr": { 2024-03-26T21:06:02.184 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 2 2024-03-26T21:06:02.184 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:06:02.184 INFO:teuthology.orchestra.run.smithi063.stdout: "osd": { 2024-03-26T21:06:02.184 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-03-26T21:06:02.184 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 4 2024-03-26T21:06:02.184 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:06:02.184 INFO:teuthology.orchestra.run.smithi063.stdout: "mds": { 2024-03-26T21:06:02.184 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-26T21:06:02.185 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:06:02.185 INFO:teuthology.orchestra.run.smithi063.stdout: "overall": { 2024-03-26T21:06:02.185 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-03-26T21:06:02.185 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 8 2024-03-26T21:06:02.185 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-03-26T21:06:02.185 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-03-26T21:06:02.521 INFO:teuthology.orchestra.run.smithi063.stdout: 5 250580 10.19 MB/sec execute 476 sec latency 66.100 ms 2024-03-26T21:06:02.833 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-03-26T21:06:02.833 INFO:teuthology.orchestra.run.smithi063.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d", 2024-03-26T21:06:02.833 INFO:teuthology.orchestra.run.smithi063.stdout: "in_progress": true, 2024-03-26T21:06:02.834 INFO:teuthology.orchestra.run.smithi063.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-26T21:06:02.834 INFO:teuthology.orchestra.run.smithi063.stdout: "services_complete": [ 2024-03-26T21:06:02.834 INFO:teuthology.orchestra.run.smithi063.stdout: "crash", 2024-03-26T21:06:02.834 INFO:teuthology.orchestra.run.smithi063.stdout: "mon", 2024-03-26T21:06:02.834 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr" 2024-03-26T21:06:02.834 INFO:teuthology.orchestra.run.smithi063.stdout: ], 2024-03-26T21:06:02.834 INFO:teuthology.orchestra.run.smithi063.stdout: "progress": "10/23 daemons upgraded", 2024-03-26T21:06:02.834 INFO:teuthology.orchestra.run.smithi063.stdout: "message": "Currently upgrading osd daemons", 2024-03-26T21:06:02.834 INFO:teuthology.orchestra.run.smithi063.stdout: "is_paused": false 2024-03-26T21:06:02.834 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-03-26T21:06:02.904 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:02 smithi063 ceph-mon[66934]: from='client.34344 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:06:02.904 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:02 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-03-26T21:06:02.904 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:02 smithi063 ceph-mon[66934]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-03-26T21:06:02.904 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:02 smithi063 ceph-mon[66934]: Upgrade: osd.4 is safe to restart 2024-03-26T21:06:02.904 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:02 smithi063 ceph-mon[66934]: from='client.? 172.21.15.63:0/2582301093' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:06:03.005 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:02 smithi174 ceph-mon[53014]: from='client.34344 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:06:03.005 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:02 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-03-26T21:06:03.005 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:02 smithi174 ceph-mon[53014]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-03-26T21:06:03.005 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:02 smithi174 ceph-mon[53014]: Upgrade: osd.4 is safe to restart 2024-03-26T21:06:03.005 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:02 smithi174 ceph-mon[53014]: from='client.? 172.21.15.63:0/2582301093' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:06:03.519 INFO:teuthology.orchestra.run.smithi063.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-26T21:06:03.520 INFO:teuthology.orchestra.run.smithi063.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-26T21:06:03.520 INFO:teuthology.orchestra.run.smithi063.stdout: daemon grafana.smithi063 on smithi063 is in error state 2024-03-26T21:06:03.521 INFO:teuthology.orchestra.run.smithi063.stdout: 5 250938 10.19 MB/sec execute 477 sec latency 72.938 ms 2024-03-26T21:06:03.930 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:03 smithi063 ceph-mon[66934]: from='client.44265 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:06:03.931 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:03 smithi063 ceph-mon[66934]: pgmap v195: 97 pgs: 97 active+clean; 331 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.1 MiB/s wr, 505 op/s; 0 B/s, 0 objects/s recovering 2024-03-26T21:06:03.931 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:03 smithi063 ceph-mon[66934]: from='client.? 172.21.15.63:0/3679417111' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-26T21:06:04.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:03 smithi174 ceph-mon[53014]: from='client.44265 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:06:04.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:03 smithi174 ceph-mon[53014]: pgmap v195: 97 pgs: 97 active+clean; 331 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.1 MiB/s wr, 505 op/s; 0 B/s, 0 objects/s recovering 2024-03-26T21:06:04.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:03 smithi174 ceph-mon[53014]: from='client.? 172.21.15.63:0/3679417111' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-26T21:06:04.521 INFO:teuthology.orchestra.run.smithi063.stdout: 5 251276 10.19 MB/sec execute 478 sec latency 37.502 ms 2024-03-26T21:06:05.226 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:04 smithi063 ceph-mon[66934]: Upgrade: Updating osd.4 2024-03-26T21:06:05.226 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:04 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:06:05.226 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:04 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-03-26T21:06:05.226 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:04 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:06:05.226 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:04 smithi063 ceph-mon[66934]: Deploying daemon osd.4 on smithi174 2024-03-26T21:06:05.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:04 smithi174 ceph-mon[53014]: Upgrade: Updating osd.4 2024-03-26T21:06:05.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:04 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:06:05.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:04 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-03-26T21:06:05.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:04 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:06:05.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:04 smithi174 ceph-mon[53014]: Deploying daemon osd.4 on smithi174 2024-03-26T21:06:05.521 INFO:teuthology.orchestra.run.smithi063.stdout: 5 251471 10.19 MB/sec execute 479 sec latency 64.627 ms 2024-03-26T21:06:05.983 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:05 smithi063 ceph-mon[66934]: pgmap v196: 97 pgs: 97 active+clean; 331 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 3.9 MiB/s wr, 438 op/s 2024-03-26T21:06:06.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:05 smithi174 ceph-mon[53014]: pgmap v196: 97 pgs: 97 active+clean; 331 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 3.9 MiB/s wr, 438 op/s 2024-03-26T21:06:06.521 INFO:teuthology.orchestra.run.smithi063.stdout: 5 251898 10.20 MB/sec execute 480 sec latency 69.358 ms 2024-03-26T21:06:07.521 INFO:teuthology.orchestra.run.smithi063.stdout: 5 252349 10.20 MB/sec execute 481 sec latency 90.321 ms 2024-03-26T21:06:08.013 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:07 smithi063 ceph-mon[66934]: pgmap v197: 97 pgs: 97 active+clean; 336 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.2 MiB/s wr, 507 op/s 2024-03-26T21:06:08.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:07 smithi174 ceph-mon[53014]: pgmap v197: 97 pgs: 97 active+clean; 336 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.2 MiB/s wr, 507 op/s 2024-03-26T21:06:08.521 INFO:teuthology.orchestra.run.smithi063.stdout: 5 252654 10.19 MB/sec execute 482 sec latency 36.238 ms 2024-03-26T21:06:08.626 INFO:journalctl@ceph.osd.4.smithi174.stdout:Mar 26 21:06:08 smithi174 systemd[1]: Stopping Ceph osd.4 for 0ed4dd4c-ebb2-11ee-95d0-87774f69a715... 2024-03-26T21:06:08.978 INFO:journalctl@ceph.osd.4.smithi174.stdout:Mar 26 21:06:08 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-4[31184]: 2024-03-26T21:06:08.631+0000 7f312898b700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-26T21:06:08.978 INFO:journalctl@ceph.osd.4.smithi174.stdout:Mar 26 21:06:08 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-4[31184]: 2024-03-26T21:06:08.631+0000 7f312898b700 -1 osd.4 117 *** Got signal Terminated *** 2024-03-26T21:06:08.978 INFO:journalctl@ceph.osd.4.smithi174.stdout:Mar 26 21:06:08 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-4[31184]: 2024-03-26T21:06:08.631+0000 7f312898b700 -1 osd.4 117 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-26T21:06:09.019 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:08 smithi063 ceph-mon[66934]: osd.4 marked itself down and dead 2024-03-26T21:06:09.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:08 smithi174 ceph-mon[53014]: osd.4 marked itself down and dead 2024-03-26T21:06:09.521 INFO:teuthology.orchestra.run.smithi063.stdout: 5 252951 10.19 MB/sec execute 483 sec latency 496.025 ms 2024-03-26T21:06:09.892 INFO:journalctl@ceph.osd.4.smithi174.stdout:Mar 26 21:06:09 smithi174 podman[57431]: 2024-03-26 21:06:09.637984622 +0000 UTC m=+1.136083477 container died a0e877ecc06d244aea0ee751290baa5c830bf4d130f8353d3668d8f370208bdf (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-4, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, io.openshift.expose-services=, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, io.buildah.version=1.19.8, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, release=754) 2024-03-26T21:06:10.240 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:09 smithi063 ceph-mon[66934]: pgmap v198: 97 pgs: 97 active+clean; 338 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 4.0 MiB/s wr, 354 op/s 2024-03-26T21:06:10.240 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:09 smithi063 ceph-mon[66934]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-26T21:06:10.240 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:09 smithi063 ceph-mon[66934]: osdmap e118: 8 total, 7 up, 8 in 2024-03-26T21:06:10.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:09 smithi174 ceph-mon[53014]: pgmap v198: 97 pgs: 97 active+clean; 338 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 4.0 MiB/s wr, 354 op/s 2024-03-26T21:06:10.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:09 smithi174 ceph-mon[53014]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-26T21:06:10.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:09 smithi174 ceph-mon[53014]: osdmap e118: 8 total, 7 up, 8 in 2024-03-26T21:06:10.298 INFO:journalctl@ceph.osd.4.smithi174.stdout:Mar 26 21:06:09 smithi174 podman[57431]: 2024-03-26 21:06:09.997499678 +0000 UTC m=+1.495598527 container cleanup a0e877ecc06d244aea0ee751290baa5c830bf4d130f8353d3668d8f370208bdf (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-4, GIT_BRANCH=HEAD, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, release=754, maintainer=Guillaume Abrioux , io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, com.redhat.component=centos-stream-container) 2024-03-26T21:06:10.298 INFO:journalctl@ceph.osd.4.smithi174.stdout:Mar 26 21:06:09 smithi174 bash[57431]: ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-4 2024-03-26T21:06:10.522 INFO:teuthology.orchestra.run.smithi063.stdout: 5 253076 10.17 MB/sec execute 484 sec latency 1024.971 ms 2024-03-26T21:06:10.625 INFO:journalctl@ceph.osd.4.smithi174.stdout:Mar 26 21:06:10 smithi174 podman[57449]: 2024-03-26 21:06:10.307467137 +0000 UTC m=+0.669371070 container remove a0e877ecc06d244aea0ee751290baa5c830bf4d130f8353d3668d8f370208bdf (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-4, com.redhat.component=centos-stream-container, distribution-scope=public, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, ceph=True, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, version=8) 2024-03-26T21:06:10.897 INFO:journalctl@ceph.osd.4.smithi174.stdout:Mar 26 21:06:10 smithi174 podman[57539]: 2024-03-26 21:06:10.535788096 +0000 UTC m=+0.020440274 image pull a9bdc3eaeb106993bcbbac35d7fe5c7b76c052021676e129f5984aed30aee063 quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d 2024-03-26T21:06:10.897 INFO:journalctl@ceph.osd.4.smithi174.stdout:Mar 26 21:06:10 smithi174 podman[57539]: 2024-03-26 21:06:10.675266963 +0000 UTC m=+0.159919129 container create d77dc9d9edd9868b9bb03ff2fd2daeccaa7501fca1501307506186b04e08b6dd (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-4-deactivate, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240326) 2024-03-26T21:06:10.897 INFO:journalctl@ceph.osd.4.smithi174.stdout:Mar 26 21:06:10 smithi174 podman[57539]: 2024-03-26 21:06:10.903520875 +0000 UTC m=+0.388173040 container init d77dc9d9edd9868b9bb03ff2fd2daeccaa7501fca1501307506186b04e08b6dd (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-4-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5) 2024-03-26T21:06:11.004 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:10 smithi063 ceph-mon[66934]: osdmap e119: 8 total, 7 up, 8 in 2024-03-26T21:06:11.004 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:10 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:06:11.004 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:10 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:06:11.248 INFO:journalctl@ceph.osd.4.smithi174.stdout:Mar 26 21:06:10 smithi174 podman[57539]: 2024-03-26 21:06:10.906977505 +0000 UTC m=+0.391629676 container start d77dc9d9edd9868b9bb03ff2fd2daeccaa7501fca1501307506186b04e08b6dd (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-4-deactivate, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, org.label-schema.build-date=20240326, ceph=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-26T21:06:11.248 INFO:journalctl@ceph.osd.4.smithi174.stdout:Mar 26 21:06:11 smithi174 podman[57539]: 2024-03-26 21:06:11.01183711 +0000 UTC m=+0.496489283 container attach d77dc9d9edd9868b9bb03ff2fd2daeccaa7501fca1501307506186b04e08b6dd (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-4-deactivate, ceph=True, org.label-schema.build-date=20240326, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b) 2024-03-26T21:06:11.248 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:10 smithi174 ceph-mon[53014]: osdmap e119: 8 total, 7 up, 8 in 2024-03-26T21:06:11.248 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:10 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:06:11.248 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:10 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:06:11.521 INFO:teuthology.orchestra.run.smithi063.stdout: 5 253455 10.16 MB/sec execute 485 sec latency 58.002 ms 2024-03-26T21:06:11.548 INFO:journalctl@ceph.osd.4.smithi174.stdout:Mar 26 21:06:11 smithi174 podman[57539]: 2024-03-26 21:06:11.254886532 +0000 UTC m=+0.739538703 container died d77dc9d9edd9868b9bb03ff2fd2daeccaa7501fca1501307506186b04e08b6dd (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-4-deactivate, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, ceph=True, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-26T21:06:12.017 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:12 smithi063 ceph-mon[66934]: pgmap v201: 97 pgs: 9 peering, 10 stale+active+clean, 78 active+clean; 342 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 2.6 KiB/s rd, 4.1 MiB/s wr, 393 op/s 2024-03-26T21:06:12.017 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:12 smithi063 ceph-mon[66934]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-03-26T21:06:12.168 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:12 smithi174 ceph-mon[53014]: pgmap v201: 97 pgs: 9 peering, 10 stale+active+clean, 78 active+clean; 342 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 2.6 KiB/s rd, 4.1 MiB/s wr, 393 op/s 2024-03-26T21:06:12.168 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:12 smithi174 ceph-mon[53014]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-03-26T21:06:12.168 INFO:journalctl@ceph.osd.4.smithi174.stdout:Mar 26 21:06:11 smithi174 podman[57565]: 2024-03-26 21:06:11.924403451 +0000 UTC m=+0.664098620 container remove d77dc9d9edd9868b9bb03ff2fd2daeccaa7501fca1501307506186b04e08b6dd (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-4-deactivate, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5) 2024-03-26T21:06:12.168 INFO:journalctl@ceph.osd.4.smithi174.stdout:Mar 26 21:06:11 smithi174 systemd[1]: ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715@osd.4.service: Deactivated successfully. 2024-03-26T21:06:12.168 INFO:journalctl@ceph.osd.4.smithi174.stdout:Mar 26 21:06:11 smithi174 systemd[1]: Stopped Ceph osd.4 for 0ed4dd4c-ebb2-11ee-95d0-87774f69a715. 2024-03-26T21:06:12.168 INFO:journalctl@ceph.osd.4.smithi174.stdout:Mar 26 21:06:11 smithi174 systemd[1]: ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715@osd.4.service: Consumed 4min 49.761s CPU time. 2024-03-26T21:06:12.168 INFO:journalctl@ceph.osd.4.smithi174.stdout:Mar 26 21:06:12 smithi174 systemd[1]: Starting Ceph osd.4 for 0ed4dd4c-ebb2-11ee-95d0-87774f69a715... 2024-03-26T21:06:12.521 INFO:teuthology.orchestra.run.smithi063.stdout: 5 253781 10.15 MB/sec execute 486 sec latency 40.003 ms 2024-03-26T21:06:12.648 INFO:journalctl@ceph.osd.4.smithi174.stdout:Mar 26 21:06:12 smithi174 podman[57667]: 2024-03-26 21:06:12.297442673 +0000 UTC m=+0.018700601 image pull a9bdc3eaeb106993bcbbac35d7fe5c7b76c052021676e129f5984aed30aee063 quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d 2024-03-26T21:06:12.648 INFO:journalctl@ceph.osd.4.smithi174.stdout:Mar 26 21:06:12 smithi174 podman[57667]: 2024-03-26 21:06:12.40880527 +0000 UTC m=+0.130063186 container create 10c7da169a4ab9ef97f930b988b5a52fc2013a12c3a7879a6d2def35ee40c54e (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-4-activate, org.label-schema.build-date=20240326, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, CEPH_POINT_RELEASE=, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-26T21:06:13.046 INFO:journalctl@ceph.osd.4.smithi174.stdout:Mar 26 21:06:12 smithi174 podman[57667]: 2024-03-26 21:06:12.654735551 +0000 UTC m=+0.375993470 container init 10c7da169a4ab9ef97f930b988b5a52fc2013a12c3a7879a6d2def35ee40c54e (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-4-activate, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-03-26T21:06:13.047 INFO:journalctl@ceph.osd.4.smithi174.stdout:Mar 26 21:06:12 smithi174 podman[57667]: 2024-03-26 21:06:12.658317242 +0000 UTC m=+0.379575168 container start 10c7da169a4ab9ef97f930b988b5a52fc2013a12c3a7879a6d2def35ee40c54e (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-4-activate, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, ceph=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0) 2024-03-26T21:06:13.047 INFO:journalctl@ceph.osd.4.smithi174.stdout:Mar 26 21:06:12 smithi174 podman[57667]: 2024-03-26 21:06:12.753890255 +0000 UTC m=+0.475148171 container attach 10c7da169a4ab9ef97f930b988b5a52fc2013a12c3a7879a6d2def35ee40c54e (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-4-activate, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5) 2024-03-26T21:06:13.063 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:13 smithi063 ceph-mon[66934]: Health check failed: Degraded data redundancy: 189/1500 objects degraded (12.600%), 24 pgs degraded (PG_DEGRADED) 2024-03-26T21:06:13.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:13 smithi174 ceph-mon[53014]: Health check failed: Degraded data redundancy: 189/1500 objects degraded (12.600%), 24 pgs degraded (PG_DEGRADED) 2024-03-26T21:06:13.298 INFO:journalctl@ceph.osd.4.smithi174.stdout:Mar 26 21:06:13 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-4-activate[57682]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-03-26T21:06:13.298 INFO:journalctl@ceph.osd.4.smithi174.stdout:Mar 26 21:06:13 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-4-activate[57682]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-03-26T21:06:13.299 INFO:journalctl@ceph.osd.4.smithi174.stdout:Mar 26 21:06:13 smithi174 bash[57667]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-03-26T21:06:13.299 INFO:journalctl@ceph.osd.4.smithi174.stdout:Mar 26 21:06:13 smithi174 bash[57667]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-03-26T21:06:13.299 INFO:journalctl@ceph.osd.4.smithi174.stdout:Mar 26 21:06:13 smithi174 bash[57667]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-03-26T21:06:13.299 INFO:journalctl@ceph.osd.4.smithi174.stdout:Mar 26 21:06:13 smithi174 bash[57667]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-03-26T21:06:13.299 INFO:journalctl@ceph.osd.4.smithi174.stdout:Mar 26 21:06:13 smithi174 bash[57667]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-03-26T21:06:13.299 INFO:journalctl@ceph.osd.4.smithi174.stdout:Mar 26 21:06:13 smithi174 bash[57667]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-03-26T21:06:13.299 INFO:journalctl@ceph.osd.4.smithi174.stdout:Mar 26 21:06:13 smithi174 bash[57667]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-03-26T21:06:13.299 INFO:journalctl@ceph.osd.4.smithi174.stdout:Mar 26 21:06:13 smithi174 bash[57667]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-03-26T21:06:13.299 INFO:journalctl@ceph.osd.4.smithi174.stdout:Mar 26 21:06:13 smithi174 bash[57667]: --> ceph-volume raw activate successful for osd ID: 4 2024-03-26T21:06:13.299 INFO:journalctl@ceph.osd.4.smithi174.stdout:Mar 26 21:06:13 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-4-activate[57682]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-03-26T21:06:13.299 INFO:journalctl@ceph.osd.4.smithi174.stdout:Mar 26 21:06:13 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-4-activate[57682]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-03-26T21:06:13.299 INFO:journalctl@ceph.osd.4.smithi174.stdout:Mar 26 21:06:13 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-4-activate[57682]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-03-26T21:06:13.299 INFO:journalctl@ceph.osd.4.smithi174.stdout:Mar 26 21:06:13 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-4-activate[57682]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-03-26T21:06:13.300 INFO:journalctl@ceph.osd.4.smithi174.stdout:Mar 26 21:06:13 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-4-activate[57682]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-03-26T21:06:13.300 INFO:journalctl@ceph.osd.4.smithi174.stdout:Mar 26 21:06:13 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-4-activate[57682]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-03-26T21:06:13.300 INFO:journalctl@ceph.osd.4.smithi174.stdout:Mar 26 21:06:13 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-4-activate[57682]: --> ceph-volume raw activate successful for osd ID: 4 2024-03-26T21:06:13.300 INFO:journalctl@ceph.osd.4.smithi174.stdout:Mar 26 21:06:13 smithi174 podman[57667]: 2024-03-26 21:06:13.216726452 +0000 UTC m=+0.937984376 container died 10c7da169a4ab9ef97f930b988b5a52fc2013a12c3a7879a6d2def35ee40c54e (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-4-activate, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5) 2024-03-26T21:06:13.521 INFO:teuthology.orchestra.run.smithi063.stdout: 5 254039 10.15 MB/sec execute 487 sec latency 56.789 ms 2024-03-26T21:06:14.078 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:14 smithi174 ceph-mon[53014]: pgmap v202: 97 pgs: 9 active+undersized, 24 active+undersized+degraded, 9 peering, 55 active+clean; 327 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 5.0 MiB/s wr, 400 op/s; 189/1500 objects degraded (12.600%) 2024-03-26T21:06:14.078 INFO:journalctl@ceph.osd.4.smithi174.stdout:Mar 26 21:06:13 smithi174 podman[57815]: 2024-03-26 21:06:13.824964594 +0000 UTC m=+0.603161910 container remove 10c7da169a4ab9ef97f930b988b5a52fc2013a12c3a7879a6d2def35ee40c54e (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-4-activate, GIT_CLEAN=True, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2) 2024-03-26T21:06:14.078 INFO:journalctl@ceph.osd.4.smithi174.stdout:Mar 26 21:06:14 smithi174 podman[57880]: 2024-03-26 21:06:13.945979649 +0000 UTC m=+0.020343604 image pull a9bdc3eaeb106993bcbbac35d7fe5c7b76c052021676e129f5984aed30aee063 quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d 2024-03-26T21:06:14.078 INFO:journalctl@ceph.osd.4.smithi174.stdout:Mar 26 21:06:14 smithi174 podman[57880]: 2024-03-26 21:06:14.084306028 +0000 UTC m=+0.158669982 container create 8aa35bff6ceace743b9eeec72019e4f30803a59ea5021a63423ea10f8b905084 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-4, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240326, io.buildah.version=1.33.5, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b) 2024-03-26T21:06:14.102 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:14 smithi063 ceph-mon[66934]: pgmap v202: 97 pgs: 9 active+undersized, 24 active+undersized+degraded, 9 peering, 55 active+clean; 327 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 5.0 MiB/s wr, 400 op/s; 189/1500 objects degraded (12.600%) 2024-03-26T21:06:14.347 INFO:journalctl@ceph.osd.4.smithi174.stdout:Mar 26 21:06:14 smithi174 podman[57880]: 2024-03-26 21:06:14.328936576 +0000 UTC m=+0.403300528 container init 8aa35bff6ceace743b9eeec72019e4f30803a59ea5021a63423ea10f8b905084 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-4, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, CEPH_POINT_RELEASE=) 2024-03-26T21:06:14.347 INFO:journalctl@ceph.osd.4.smithi174.stdout:Mar 26 21:06:14 smithi174 podman[57880]: 2024-03-26 21:06:14.332480254 +0000 UTC m=+0.406844209 container start 8aa35bff6ceace743b9eeec72019e4f30803a59ea5021a63423ea10f8b905084 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-4, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b) 2024-03-26T21:06:14.521 INFO:teuthology.orchestra.run.smithi063.stdout: 5 254438 10.16 MB/sec execute 488 sec latency 40.574 ms 2024-03-26T21:06:14.798 INFO:journalctl@ceph.osd.4.smithi174.stdout:Mar 26 21:06:14 smithi174 ceph-osd[57899]: -- 172.21.15.174:0/169642588 <== mon.1 v2:172.21.15.174:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x559b63991680 con 0x559b6395ac00 2024-03-26T21:06:14.798 INFO:journalctl@ceph.osd.4.smithi174.stdout:Mar 26 21:06:14 smithi174 bash[57880]: 8aa35bff6ceace743b9eeec72019e4f30803a59ea5021a63423ea10f8b905084 2024-03-26T21:06:14.798 INFO:journalctl@ceph.osd.4.smithi174.stdout:Mar 26 21:06:14 smithi174 systemd[1]: Started Ceph osd.4 for 0ed4dd4c-ebb2-11ee-95d0-87774f69a715. 2024-03-26T21:06:15.298 INFO:journalctl@ceph.osd.4.smithi174.stdout:Mar 26 21:06:14 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-4[57895]: 2024-03-26T21:06:14.876+0000 7f88bc8e77c0 -1 Falling back to public interface 2024-03-26T21:06:15.521 INFO:teuthology.orchestra.run.smithi063.stdout: 5 254755 10.15 MB/sec execute 489 sec latency 70.580 ms 2024-03-26T21:06:15.637 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:15 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:06:15.637 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:15 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:06:15.637 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:15 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:06:15.637 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:15 smithi063 ceph-mon[66934]: pgmap v203: 97 pgs: 11 active+undersized, 28 active+undersized+degraded, 3 peering, 55 active+clean; 327 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 3.4 MiB/s wr, 375 op/s; 220/1479 objects degraded (14.875%) 2024-03-26T21:06:15.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:15 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:06:15.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:15 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:06:15.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:15 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:06:15.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:15 smithi174 ceph-mon[53014]: pgmap v203: 97 pgs: 11 active+undersized, 28 active+undersized+degraded, 3 peering, 55 active+clean; 327 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 3.4 MiB/s wr, 375 op/s; 220/1479 objects degraded (14.875%) 2024-03-26T21:06:16.521 INFO:teuthology.orchestra.run.smithi063.stdout: 5 255049 10.15 MB/sec execute 490 sec latency 52.879 ms 2024-03-26T21:06:17.160 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:17 smithi063 ceph-mon[66934]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-03-26T21:06:17.352 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:17 smithi174 ceph-mon[53014]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-03-26T21:06:17.521 INFO:teuthology.orchestra.run.smithi063.stdout: 5 255318 10.16 MB/sec execute 491 sec latency 59.197 ms 2024-03-26T21:06:18.178 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:18 smithi063 ceph-mon[66934]: pgmap v204: 97 pgs: 13 active+undersized, 29 active+undersized+degraded, 55 active+clean; 335 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 4.9 MiB/s wr, 558 op/s; 253/1587 objects degraded (15.942%) 2024-03-26T21:06:18.521 INFO:teuthology.orchestra.run.smithi063.stdout: 5 255556 10.15 MB/sec execute 492 sec latency 51.965 ms 2024-03-26T21:06:18.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:18 smithi174 ceph-mon[53014]: pgmap v204: 97 pgs: 13 active+undersized, 29 active+undersized+degraded, 55 active+clean; 335 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 4.9 MiB/s wr, 558 op/s; 253/1587 objects degraded (15.942%) 2024-03-26T21:06:19.521 INFO:teuthology.orchestra.run.smithi063.stdout: 5 255824 10.15 MB/sec execute 493 sec latency 63.647 ms 2024-03-26T21:06:19.723 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:19 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:06:19.723 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:19 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:06:19.723 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:19 smithi063 ceph-mon[66934]: pgmap v205: 97 pgs: 13 active+undersized, 29 active+undersized+degraded, 55 active+clean; 338 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.4 MiB/s wr, 468 op/s; 261/1626 objects degraded (16.052%) 2024-03-26T21:06:19.723 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:19 smithi063 ceph-mon[66934]: Health check update: Degraded data redundancy: 253/1587 objects degraded (15.942%), 29 pgs degraded (PG_DEGRADED) 2024-03-26T21:06:20.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:19 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:06:20.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:19 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:06:20.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:19 smithi174 ceph-mon[53014]: pgmap v205: 97 pgs: 13 active+undersized, 29 active+undersized+degraded, 55 active+clean; 338 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.4 MiB/s wr, 468 op/s; 261/1626 objects degraded (16.052%) 2024-03-26T21:06:20.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:19 smithi174 ceph-mon[53014]: Health check update: Degraded data redundancy: 253/1587 objects degraded (15.942%), 29 pgs degraded (PG_DEGRADED) 2024-03-26T21:06:20.522 INFO:teuthology.orchestra.run.smithi063.stdout: 5 256119 10.13 MB/sec execute 494 sec latency 35.697 ms 2024-03-26T21:06:21.255 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:21 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:06:21.256 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:21 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:06:21.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:21 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:06:21.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:21 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:06:21.521 INFO:teuthology.orchestra.run.smithi063.stdout: 5 256508 10.14 MB/sec execute 495 sec latency 51.649 ms 2024-03-26T21:06:22.272 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:22 smithi063 ceph-mon[66934]: pgmap v206: 97 pgs: 13 active+undersized, 29 active+undersized+degraded, 55 active+clean; 335 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 3.7 MiB/s wr, 386 op/s; 262/1635 objects degraded (16.024%) 2024-03-26T21:06:22.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:22 smithi174 ceph-mon[53014]: pgmap v206: 97 pgs: 13 active+undersized, 29 active+undersized+degraded, 55 active+clean; 335 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 3.7 MiB/s wr, 386 op/s; 262/1635 objects degraded (16.024%) 2024-03-26T21:06:22.298 INFO:journalctl@ceph.osd.4.smithi174.stdout:Mar 26 21:06:22 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-4[57895]: 2024-03-26T21:06:22.047+0000 7f88bc8e77c0 -1 osd.4 0 read_superblock omap replica is missing. 2024-03-26T21:06:22.521 INFO:teuthology.orchestra.run.smithi063.stdout: 5 256969 10.13 MB/sec execute 496 sec latency 39.102 ms 2024-03-26T21:06:23.521 INFO:teuthology.orchestra.run.smithi063.stdout: 5 257458 10.13 MB/sec execute 497 sec latency 26.699 ms 2024-03-26T21:06:24.038 INFO:journalctl@ceph.osd.4.smithi174.stdout:Mar 26 21:06:23 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-4[57895]: 2024-03-26T21:06:23.579+0000 7f88bc8e77c0 -1 osd.4 117 log_to_monitors true 2024-03-26T21:06:24.286 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:24 smithi063 ceph-mon[66934]: pgmap v207: 97 pgs: 13 active+undersized, 29 active+undersized+degraded, 55 active+clean; 344 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 5.3 MiB/s wr, 547 op/s; 262/1659 objects degraded (15.793%) 2024-03-26T21:06:24.286 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:24 smithi063 ceph-mon[66934]: from='osd.4 [v2:172.21.15.174:6800/3833826902,v1:172.21.15.174:6801/3833826902]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-26T21:06:24.286 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:24 smithi063 ceph-mon[66934]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-26T21:06:24.287 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:24 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:06:24.287 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:24 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:06:24.287 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:24 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:06:24.287 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:24 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T21:06:24.287 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:24 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:06:24.287 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:24 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:06:24.287 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:24 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:06:24.287 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:24 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:06:24.287 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:24 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:06:24.287 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:24 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:06:24.287 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:24 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-03-26T21:06:24.298 INFO:journalctl@ceph.osd.4.smithi174.stdout:Mar 26 21:06:24 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-4[57895]: 2024-03-26T21:06:24.074+0000 7f88add3f700 -1 osd.4 117 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-26T21:06:24.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:24 smithi174 ceph-mon[53014]: pgmap v207: 97 pgs: 13 active+undersized, 29 active+undersized+degraded, 55 active+clean; 344 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 5.3 MiB/s wr, 547 op/s; 262/1659 objects degraded (15.793%) 2024-03-26T21:06:24.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:24 smithi174 ceph-mon[53014]: from='osd.4 [v2:172.21.15.174:6800/3833826902,v1:172.21.15.174:6801/3833826902]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-26T21:06:24.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:24 smithi174 ceph-mon[53014]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-26T21:06:24.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:24 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:06:24.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:24 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:06:24.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:24 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:06:24.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:24 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T21:06:24.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:24 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:06:24.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:24 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:06:24.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:24 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:06:24.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:24 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:06:24.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:24 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:06:24.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:24 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:06:24.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:24 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-03-26T21:06:24.522 INFO:teuthology.orchestra.run.smithi063.stdout: 5 258143 10.15 MB/sec execute 498 sec latency 20.885 ms 2024-03-26T21:06:25.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:25 smithi063 ceph-mon[66934]: pgmap v208: 97 pgs: 13 active+undersized, 29 active+undersized+degraded, 55 active+clean; 344 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 4.7 MiB/s wr, 485 op/s; 262/1659 objects degraded (15.793%) 2024-03-26T21:06:25.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:25 smithi063 ceph-mon[66934]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-03-26T21:06:25.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:25 smithi063 ceph-mon[66934]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-03-26T21:06:25.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:25 smithi063 ceph-mon[66934]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-03-26T21:06:25.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:25 smithi063 ceph-mon[66934]: from='osd.4 [v2:172.21.15.174:6800/3833826902,v1:172.21.15.174:6801/3833826902]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]: dispatch 2024-03-26T21:06:25.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:25 smithi063 ceph-mon[66934]: osdmap e120: 8 total, 7 up, 8 in 2024-03-26T21:06:25.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:25 smithi063 ceph-mon[66934]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]: dispatch 2024-03-26T21:06:25.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:25 smithi063 ceph-mon[66934]: Health check update: Degraded data redundancy: 262/1659 objects degraded (15.793%), 29 pgs degraded (PG_DEGRADED) 2024-03-26T21:06:25.522 INFO:teuthology.orchestra.run.smithi063.stdout: 5 258384 10.14 MB/sec execute 499 sec latency 481.744 ms 2024-03-26T21:06:25.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:25 smithi174 ceph-mon[53014]: pgmap v208: 97 pgs: 13 active+undersized, 29 active+undersized+degraded, 55 active+clean; 344 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 4.7 MiB/s wr, 485 op/s; 262/1659 objects degraded (15.793%) 2024-03-26T21:06:25.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:25 smithi174 ceph-mon[53014]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-03-26T21:06:25.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:25 smithi174 ceph-mon[53014]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-03-26T21:06:25.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:25 smithi174 ceph-mon[53014]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-03-26T21:06:25.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:25 smithi174 ceph-mon[53014]: from='osd.4 [v2:172.21.15.174:6800/3833826902,v1:172.21.15.174:6801/3833826902]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]: dispatch 2024-03-26T21:06:25.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:25 smithi174 ceph-mon[53014]: osdmap e120: 8 total, 7 up, 8 in 2024-03-26T21:06:25.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:25 smithi174 ceph-mon[53014]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]: dispatch 2024-03-26T21:06:25.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:25 smithi174 ceph-mon[53014]: Health check update: Degraded data redundancy: 262/1659 objects degraded (15.793%), 29 pgs degraded (PG_DEGRADED) 2024-03-26T21:06:26.317 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:26 smithi063 ceph-mon[66934]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-26T21:06:26.317 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:26 smithi063 ceph-mon[66934]: osd.4 [v2:172.21.15.174:6800/3833826902,v1:172.21.15.174:6801/3833826902] boot 2024-03-26T21:06:26.317 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:26 smithi063 ceph-mon[66934]: osdmap e121: 8 total, 8 up, 8 in 2024-03-26T21:06:26.317 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:26 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-26T21:06:26.317 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:26 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:06:26.317 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:26 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:06:26.522 INFO:teuthology.orchestra.run.smithi063.stdout: 5 258440 10.13 MB/sec execute 500 sec latency 1137.813 ms 2024-03-26T21:06:26.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:26 smithi174 ceph-mon[53014]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-26T21:06:26.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:26 smithi174 ceph-mon[53014]: osd.4 [v2:172.21.15.174:6800/3833826902,v1:172.21.15.174:6801/3833826902] boot 2024-03-26T21:06:26.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:26 smithi174 ceph-mon[53014]: osdmap e121: 8 total, 8 up, 8 in 2024-03-26T21:06:26.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:26 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-26T21:06:26.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:26 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:06:26.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:26 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:06:27.317 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:27 smithi063 ceph-mon[66934]: pgmap v211: 97 pgs: 13 active+undersized, 29 active+undersized+degraded, 55 active+clean; 339 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 4.3 MiB/s wr, 431 op/s; 267/1659 objects degraded (16.094%) 2024-03-26T21:06:27.317 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:27 smithi063 ceph-mon[66934]: osdmap e122: 8 total, 8 up, 8 in 2024-03-26T21:06:27.317 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:27 smithi063 ceph-mon[66934]: osdmap e123: 8 total, 8 up, 8 in 2024-03-26T21:06:27.522 INFO:teuthology.orchestra.run.smithi063.stdout: 5 258549 10.12 MB/sec execute 501 sec latency 1033.051 ms 2024-03-26T21:06:27.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:27 smithi174 ceph-mon[53014]: pgmap v211: 97 pgs: 13 active+undersized, 29 active+undersized+degraded, 55 active+clean; 339 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 4.3 MiB/s wr, 431 op/s; 267/1659 objects degraded (16.094%) 2024-03-26T21:06:27.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:27 smithi174 ceph-mon[53014]: osdmap e122: 8 total, 8 up, 8 in 2024-03-26T21:06:27.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:27 smithi174 ceph-mon[53014]: osdmap e123: 8 total, 8 up, 8 in 2024-03-26T21:06:28.522 INFO:teuthology.orchestra.run.smithi063.stdout: 5 258854 10.13 MB/sec execute 502 sec latency 47.858 ms 2024-03-26T21:06:29.115 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:29 smithi063 ceph-mon[66934]: pgmap v214: 97 pgs: 2 activating+undersized+degraded+remapped, 2 activating+undersized+remapped, 1 active+recovering+degraded, 1 active+recovery_wait+degraded, 5 remapped+peering, 11 active+undersized, 7 active+undersized+degraded, 68 active+clean; 334 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.2 MiB/s wr, 652 op/s; 144/1617 objects degraded (8.905%); 100 KiB/s, 17 objects/s recovering 2024-03-26T21:06:29.115 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:29 smithi063 ceph-mon[66934]: osdmap e124: 8 total, 8 up, 8 in 2024-03-26T21:06:29.350 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:29 smithi174 ceph-mon[53014]: pgmap v214: 97 pgs: 2 activating+undersized+degraded+remapped, 2 activating+undersized+remapped, 1 active+recovering+degraded, 1 active+recovery_wait+degraded, 5 remapped+peering, 11 active+undersized, 7 active+undersized+degraded, 68 active+clean; 334 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.2 MiB/s wr, 652 op/s; 144/1617 objects degraded (8.905%); 100 KiB/s, 17 objects/s recovering 2024-03-26T21:06:29.350 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:29 smithi174 ceph-mon[53014]: osdmap e124: 8 total, 8 up, 8 in 2024-03-26T21:06:29.525 INFO:teuthology.orchestra.run.smithi063.stdout: 5 259115 10.12 MB/sec execute 503 sec latency 409.700 ms 2024-03-26T21:06:30.151 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:30 smithi063 ceph-mon[66934]: osdmap e125: 8 total, 8 up, 8 in 2024-03-26T21:06:30.151 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:30 smithi063 ceph-mon[66934]: Health check update: Degraded data redundancy: 144/1617 objects degraded (8.905%), 11 pgs degraded (PG_DEGRADED) 2024-03-26T21:06:30.525 INFO:teuthology.orchestra.run.smithi063.stdout: 5 259327 10.10 MB/sec execute 504 sec latency 1027.690 ms 2024-03-26T21:06:30.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:30 smithi174 ceph-mon[53014]: osdmap e125: 8 total, 8 up, 8 in 2024-03-26T21:06:30.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:30 smithi174 ceph-mon[53014]: Health check update: Degraded data redundancy: 144/1617 objects degraded (8.905%), 11 pgs degraded (PG_DEGRADED) 2024-03-26T21:06:31.184 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:31 smithi063 ceph-mon[66934]: pgmap v217: 97 pgs: 1 peering, 2 activating+undersized+degraded+remapped, 1 activating+undersized+remapped, 1 active+recovering+degraded, 1 active+recovery_wait+degraded, 5 remapped+peering, 1 active+undersized+degraded, 85 active+clean; 335 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.0 MiB/s wr, 631 op/s; 89/1614 objects degraded (5.514%); 176 KiB/s, 55 keys/s, 34 objects/s recovering 2024-03-26T21:06:31.185 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:31 smithi063 ceph-mon[66934]: osdmap e126: 8 total, 8 up, 8 in 2024-03-26T21:06:31.525 INFO:teuthology.orchestra.run.smithi063.stdout: 5 259560 10.10 MB/sec execute 505 sec latency 55.516 ms 2024-03-26T21:06:31.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:31 smithi174 ceph-mon[53014]: pgmap v217: 97 pgs: 1 peering, 2 activating+undersized+degraded+remapped, 1 activating+undersized+remapped, 1 active+recovering+degraded, 1 active+recovery_wait+degraded, 5 remapped+peering, 1 active+undersized+degraded, 85 active+clean; 335 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.0 MiB/s wr, 631 op/s; 89/1614 objects degraded (5.514%); 176 KiB/s, 55 keys/s, 34 objects/s recovering 2024-03-26T21:06:31.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:31 smithi174 ceph-mon[53014]: osdmap e126: 8 total, 8 up, 8 in 2024-03-26T21:06:32.205 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:32 smithi063 ceph-mon[66934]: osdmap e127: 8 total, 8 up, 8 in 2024-03-26T21:06:32.525 INFO:teuthology.orchestra.run.smithi063.stdout: 5 259732 10.09 MB/sec execute 506 sec latency 51.016 ms 2024-03-26T21:06:32.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:32 smithi174 ceph-mon[53014]: osdmap e127: 8 total, 8 up, 8 in 2024-03-26T21:06:33.230 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:33 smithi063 ceph-mon[66934]: pgmap v220: 97 pgs: 1 peering, 2 activating+undersized+degraded+remapped, 1 activating+undersized+remapped, 1 active+undersized+degraded, 92 active+clean; 341 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 3.7 MiB/s wr, 388 op/s; 67/1635 objects degraded (4.098%); 87 KiB/s, 55 keys/s, 57 objects/s recovering 2024-03-26T21:06:33.525 INFO:teuthology.orchestra.run.smithi063.stdout: 5 259965 10.08 MB/sec execute 507 sec latency 60.718 ms 2024-03-26T21:06:33.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:33 smithi174 ceph-mon[53014]: pgmap v220: 97 pgs: 1 peering, 2 activating+undersized+degraded+remapped, 1 activating+undersized+remapped, 1 active+undersized+degraded, 92 active+clean; 341 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 3.7 MiB/s wr, 388 op/s; 67/1635 objects degraded (4.098%); 87 KiB/s, 55 keys/s, 57 objects/s recovering 2024-03-26T21:06:34.011 INFO:teuthology.orchestra.run.smithi063.stdout:true 2024-03-26T21:06:34.525 INFO:teuthology.orchestra.run.smithi063.stdout: 5 260290 10.07 MB/sec execute 508 sec latency 69.021 ms 2024-03-26T21:06:34.994 INFO:teuthology.orchestra.run.smithi063.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-26T21:06:34.994 INFO:teuthology.orchestra.run.smithi063.stdout:alertmanager.smithi063 smithi063 *:9093,9094 running (7m) 54s ago 16m 24.5M - 0.23.0 ba2b418f427c 6e160c94aac3 2024-03-26T21:06:34.994 INFO:teuthology.orchestra.run.smithi063.stdout:crash.smithi063 smithi063 running (3m) 54s ago 16m 6665k - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 cf1096d84270 2024-03-26T21:06:34.994 INFO:teuthology.orchestra.run.smithi063.stdout:crash.smithi174 smithi174 running (3m) 16s ago 16m 6665k - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 cb5371a92684 2024-03-26T21:06:34.994 INFO:teuthology.orchestra.run.smithi063.stdout:grafana.smithi063 smithi063 *:3000 error 54s ago 16m - - 2024-03-26T21:06:34.994 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foofs.smithi063.tzaznz smithi063 running (11m) 54s ago 10m 37.8M - 17.2.0 e1d6a67b021e 10a989fc7954 2024-03-26T21:06:34.994 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foofs.smithi174.gitdvk smithi174 running (10m) 16s ago 10m 14.1M - 17.2.0 e1d6a67b021e 4339a5d98d1d 2024-03-26T21:06:34.994 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.smithi063.lxhujt smithi063 *:8443,9283,8765 running (6m) 54s ago 18m 509M - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 6d9d10330a90 2024-03-26T21:06:34.995 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.smithi174.sexoin smithi174 *:8443,9283,8765 running (5m) 16s ago 16m 452M - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 d7546233672d 2024-03-26T21:06:34.995 INFO:teuthology.orchestra.run.smithi063.stdout:mon.smithi063 smithi063 running (5m) 54s ago 18m 52.3M 2048M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 c644200344f8 2024-03-26T21:06:34.995 INFO:teuthology.orchestra.run.smithi063.stdout:mon.smithi174 smithi174 running (4m) 16s ago 16m 42.5M 2048M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 1a6d442692e7 2024-03-26T21:06:34.995 INFO:teuthology.orchestra.run.smithi063.stdout:nfs.foo.0.0.smithi063.fgmpwa smithi063 *:2049 running (10m) 54s ago 10m 275M - 4.0 e1d6a67b021e 3d6c0dd9ce51 2024-03-26T21:06:34.995 INFO:teuthology.orchestra.run.smithi063.stdout:nfs.foo.1.0.smithi174.pytptt smithi174 *:2049 running (10m) 16s ago 10m 86.9M - 4.0 e1d6a67b021e f220dbb449a3 2024-03-26T21:06:34.995 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.smithi063 smithi063 *:9100 running (7m) 54s ago 16m 20.7M - 1.3.1 1dbe0e931976 f52f41fd14c0 2024-03-26T21:06:34.995 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.smithi174 smithi174 *:9100 running (6m) 16s ago 16m 22.1M - 1.3.1 1dbe0e931976 0c027f39a4c8 2024-03-26T21:06:34.995 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0 smithi063 running (3m) 54s ago 15m 339M 1936M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 2d4b3fa825cf 2024-03-26T21:06:34.995 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1 smithi063 running (2m) 54s ago 14m 282M 1936M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 d41fbe507197 2024-03-26T21:06:34.995 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2 smithi063 running (99s) 54s ago 14m 200M 1936M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 f123eeabd3fa 2024-03-26T21:06:34.995 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3 smithi063 running (65s) 54s ago 13m 15.3M 1936M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 953816eafe87 2024-03-26T21:06:34.995 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4 smithi174 running (21s) 16s ago 13m 12.7M 2704M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 8aa35bff6cea 2024-03-26T21:06:34.995 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5 smithi174 running (13m) 16s ago 13m 599M 2704M 17.2.0 e1d6a67b021e 378c2aca7e71 2024-03-26T21:06:34.995 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6 smithi174 running (12m) 16s ago 12m 524M 2704M 17.2.0 e1d6a67b021e ffbb5b7ad3b6 2024-03-26T21:06:34.996 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7 smithi174 running (12m) 16s ago 12m 771M 2704M 17.2.0 e1d6a67b021e c1d57fcabb3f 2024-03-26T21:06:34.996 INFO:teuthology.orchestra.run.smithi063.stdout:prometheus.smithi063 smithi063 *:9095 running (5m) 54s ago 15m 76.0M - 2.33.4 514e6a882f6e 567eef202627 2024-03-26T21:06:35.342 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:35 smithi063 ceph-mon[66934]: pgmap v221: 97 pgs: 1 peering, 1 active+undersized+degraded, 95 active+clean; 341 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.3 MiB/s wr, 299 op/s; 11/1596 objects degraded (0.689%); 984 KiB/s, 39 keys/s, 50 objects/s recovering 2024-03-26T21:06:35.342 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:35 smithi063 ceph-mon[66934]: from='client.34362 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:06:35.342 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:35 smithi063 ceph-mon[66934]: Health check update: Degraded data redundancy: 11/1596 objects degraded (0.689%), 1 pg degraded (PG_DEGRADED) 2024-03-26T21:06:35.525 INFO:teuthology.orchestra.run.smithi063.stdout: 5 260707 10.06 MB/sec execute 509 sec latency 46.836 ms 2024-03-26T21:06:35.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:35 smithi174 ceph-mon[53014]: pgmap v221: 97 pgs: 1 peering, 1 active+undersized+degraded, 95 active+clean; 341 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.3 MiB/s wr, 299 op/s; 11/1596 objects degraded (0.689%); 984 KiB/s, 39 keys/s, 50 objects/s recovering 2024-03-26T21:06:35.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:35 smithi174 ceph-mon[53014]: from='client.34362 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:06:35.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:35 smithi174 ceph-mon[53014]: Health check update: Degraded data redundancy: 11/1596 objects degraded (0.689%), 1 pg degraded (PG_DEGRADED) 2024-03-26T21:06:35.596 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-03-26T21:06:35.597 INFO:teuthology.orchestra.run.smithi063.stdout: "mon": { 2024-03-26T21:06:35.597 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 2 2024-03-26T21:06:35.597 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:06:35.597 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr": { 2024-03-26T21:06:35.597 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 2 2024-03-26T21:06:35.597 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:06:35.597 INFO:teuthology.orchestra.run.smithi063.stdout: "osd": { 2024-03-26T21:06:35.597 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-03-26T21:06:35.597 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 5 2024-03-26T21:06:35.597 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:06:35.597 INFO:teuthology.orchestra.run.smithi063.stdout: "mds": { 2024-03-26T21:06:35.597 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-26T21:06:35.598 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:06:35.598 INFO:teuthology.orchestra.run.smithi063.stdout: "overall": { 2024-03-26T21:06:35.598 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-03-26T21:06:35.598 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 9 2024-03-26T21:06:35.598 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-03-26T21:06:35.598 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-03-26T21:06:36.113 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-03-26T21:06:36.113 INFO:teuthology.orchestra.run.smithi063.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d", 2024-03-26T21:06:36.113 INFO:teuthology.orchestra.run.smithi063.stdout: "in_progress": true, 2024-03-26T21:06:36.113 INFO:teuthology.orchestra.run.smithi063.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-26T21:06:36.113 INFO:teuthology.orchestra.run.smithi063.stdout: "services_complete": [ 2024-03-26T21:06:36.113 INFO:teuthology.orchestra.run.smithi063.stdout: "crash", 2024-03-26T21:06:36.113 INFO:teuthology.orchestra.run.smithi063.stdout: "mon", 2024-03-26T21:06:36.113 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr" 2024-03-26T21:06:36.114 INFO:teuthology.orchestra.run.smithi063.stdout: ], 2024-03-26T21:06:36.114 INFO:teuthology.orchestra.run.smithi063.stdout: "progress": "11/23 daemons upgraded", 2024-03-26T21:06:36.114 INFO:teuthology.orchestra.run.smithi063.stdout: "message": "Currently upgrading osd daemons", 2024-03-26T21:06:36.114 INFO:teuthology.orchestra.run.smithi063.stdout: "is_paused": false 2024-03-26T21:06:36.114 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-03-26T21:06:36.356 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:36 smithi063 ceph-mon[66934]: from='client.34366 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:06:36.356 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:36 smithi063 ceph-mon[66934]: from='client.34370 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:06:36.356 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:36 smithi063 ceph-mon[66934]: from='client.? 172.21.15.63:0/834530613' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:06:36.525 INFO:teuthology.orchestra.run.smithi063.stdout: 5 261020 10.06 MB/sec execute 510 sec latency 53.818 ms 2024-03-26T21:06:36.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:36 smithi174 ceph-mon[53014]: from='client.34366 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:06:36.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:36 smithi174 ceph-mon[53014]: from='client.34370 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:06:36.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:36 smithi174 ceph-mon[53014]: from='client.? 172.21.15.63:0/834530613' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:06:36.687 INFO:teuthology.orchestra.run.smithi063.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-26T21:06:36.688 INFO:teuthology.orchestra.run.smithi063.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-26T21:06:36.688 INFO:teuthology.orchestra.run.smithi063.stdout: daemon grafana.smithi063 on smithi063 is in error state 2024-03-26T21:06:37.117 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:37 smithi063 ceph-mon[66934]: pgmap v222: 97 pgs: 97 active+clean; 336 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.4 KiB/s rd, 3.0 MiB/s wr, 264 op/s; 1.5 MiB/s, 33 keys/s, 38 objects/s recovering 2024-03-26T21:06:37.117 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:37 smithi063 ceph-mon[66934]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 11/1596 objects degraded (0.689%), 1 pg degraded) 2024-03-26T21:06:37.117 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:37 smithi063 ceph-mon[66934]: from='client.34378 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:06:37.117 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:37 smithi063 ceph-mon[66934]: from='client.? 172.21.15.63:0/2568119000' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-26T21:06:37.525 INFO:teuthology.orchestra.run.smithi063.stdout: 5 261411 10.07 MB/sec execute 511 sec latency 73.971 ms 2024-03-26T21:06:37.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:37 smithi174 ceph-mon[53014]: pgmap v222: 97 pgs: 97 active+clean; 336 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.4 KiB/s rd, 3.0 MiB/s wr, 264 op/s; 1.5 MiB/s, 33 keys/s, 38 objects/s recovering 2024-03-26T21:06:37.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:37 smithi174 ceph-mon[53014]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 11/1596 objects degraded (0.689%), 1 pg degraded) 2024-03-26T21:06:37.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:37 smithi174 ceph-mon[53014]: from='client.34378 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:06:37.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:37 smithi174 ceph-mon[53014]: from='client.? 172.21.15.63:0/2568119000' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-26T21:06:38.526 INFO:teuthology.orchestra.run.smithi063.stdout: 5 261802 10.07 MB/sec execute 512 sec latency 81.551 ms 2024-03-26T21:06:39.235 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:39 smithi063 ceph-mon[66934]: pgmap v223: 97 pgs: 97 active+clean; 333 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 4.6 MiB/s wr, 495 op/s; 1.2 MiB/s, 28 objects/s recovering 2024-03-26T21:06:39.235 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:39 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-03-26T21:06:39.525 INFO:teuthology.orchestra.run.smithi063.stdout: 5 262095 10.07 MB/sec execute 513 sec latency 58.517 ms 2024-03-26T21:06:39.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:39 smithi174 ceph-mon[53014]: pgmap v223: 97 pgs: 97 active+clean; 333 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 4.6 MiB/s wr, 495 op/s; 1.2 MiB/s, 28 objects/s recovering 2024-03-26T21:06:39.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:39 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-03-26T21:06:40.148 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:40 smithi174 ceph-mon[53014]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-03-26T21:06:40.148 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:40 smithi174 ceph-mon[53014]: Upgrade: osd.5 is safe to restart 2024-03-26T21:06:40.270 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:40 smithi063 ceph-mon[66934]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-03-26T21:06:40.270 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:40 smithi063 ceph-mon[66934]: Upgrade: osd.5 is safe to restart 2024-03-26T21:06:40.525 INFO:teuthology.orchestra.run.smithi063.stdout: 5 262278 10.06 MB/sec execute 514 sec latency 53.474 ms 2024-03-26T21:06:41.292 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:41 smithi063 ceph-mon[66934]: pgmap v224: 97 pgs: 97 active+clean; 332 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.0 MiB/s wr, 425 op/s; 1.0 MiB/s, 24 objects/s recovering 2024-03-26T21:06:41.292 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:41 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:06:41.292 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:41 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:06:41.292 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:41 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:06:41.292 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:41 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:06:41.292 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:41 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-03-26T21:06:41.292 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:41 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:06:41.527 INFO:teuthology.orchestra.run.smithi063.stdout: 5 262524 10.06 MB/sec execute 515 sec latency 69.228 ms 2024-03-26T21:06:41.536 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:41 smithi174 ceph-mon[53014]: pgmap v224: 97 pgs: 97 active+clean; 332 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.0 MiB/s wr, 425 op/s; 1.0 MiB/s, 24 objects/s recovering 2024-03-26T21:06:41.536 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:41 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:06:41.536 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:41 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:06:41.536 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:41 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:06:41.536 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:41 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:06:41.536 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:41 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-03-26T21:06:41.537 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:41 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:06:42.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:42 smithi063 ceph-mon[66934]: Upgrade: Updating osd.5 2024-03-26T21:06:42.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:42 smithi063 ceph-mon[66934]: Deploying daemon osd.5 on smithi174 2024-03-26T21:06:42.527 INFO:teuthology.orchestra.run.smithi063.stdout: 5 262910 10.05 MB/sec execute 516 sec latency 51.406 ms 2024-03-26T21:06:42.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:42 smithi174 ceph-mon[53014]: Upgrade: Updating osd.5 2024-03-26T21:06:42.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:42 smithi174 ceph-mon[53014]: Deploying daemon osd.5 on smithi174 2024-03-26T21:06:43.320 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:43 smithi063 ceph-mon[66934]: pgmap v225: 97 pgs: 97 active+clean; 334 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.2 MiB/s wr, 552 op/s; 933 KiB/s, 21 objects/s recovering 2024-03-26T21:06:43.378 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:43 smithi174 ceph-mon[53014]: pgmap v225: 97 pgs: 97 active+clean; 334 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.2 MiB/s wr, 552 op/s; 933 KiB/s, 21 objects/s recovering 2024-03-26T21:06:43.527 INFO:teuthology.orchestra.run.smithi063.stdout: 5 263255 10.06 MB/sec execute 517 sec latency 35.133 ms 2024-03-26T21:06:44.527 INFO:teuthology.orchestra.run.smithi063.stdout: 5 263607 10.06 MB/sec execute 518 sec latency 43.447 ms 2024-03-26T21:06:45.242 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:45 smithi174 ceph-mon[53014]: pgmap v226: 97 pgs: 97 active+clean; 336 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.9 MiB/s wr, 381 op/s; 822 KiB/s, 6 objects/s recovering 2024-03-26T21:06:45.243 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:06:44 smithi174 systemd[1]: Stopping Ceph osd.5 for 0ed4dd4c-ebb2-11ee-95d0-87774f69a715... 2024-03-26T21:06:45.347 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:45 smithi063 ceph-mon[66934]: pgmap v226: 97 pgs: 97 active+clean; 336 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.9 MiB/s wr, 381 op/s; 822 KiB/s, 6 objects/s recovering 2024-03-26T21:06:45.527 INFO:teuthology.orchestra.run.smithi063.stdout: 5 263935 10.04 MB/sec execute 519 sec latency 90.080 ms 2024-03-26T21:06:45.548 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:06:45 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-5[34148]: 2024-03-26T21:06:45.247+0000 7f435f38a700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-26T21:06:45.548 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:06:45 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-5[34148]: 2024-03-26T21:06:45.247+0000 7f435f38a700 -1 osd.5 127 *** Got signal Terminated *** 2024-03-26T21:06:45.548 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:06:45 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-5[34148]: 2024-03-26T21:06:45.247+0000 7f435f38a700 -1 osd.5 127 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-26T21:06:46.527 INFO:teuthology.orchestra.run.smithi063.stdout: 5 264177 10.03 MB/sec execute 520 sec latency 376.966 ms 2024-03-26T21:06:46.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:46 smithi174 ceph-mon[53014]: osd.5 marked itself down and dead 2024-03-26T21:06:46.551 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:46 smithi063 ceph-mon[66934]: osd.5 marked itself down and dead 2024-03-26T21:06:47.075 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:06:46 smithi174 podman[59708]: 2024-03-26 21:06:46.815665944 +0000 UTC m=+1.691246987 container died 378c2aca7e71b7662f4b2639a030b3313e92e4d4bfef7b701d20daa65b470db8 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-5, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, version=8, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, io.k8s.display-name=CentOS Stream 8, vcs-type=git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, io.buildah.version=1.19.8, GIT_BRANCH=HEAD) 2024-03-26T21:06:47.431 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:47 smithi063 ceph-mon[66934]: pgmap v227: 97 pgs: 97 active+clean; 338 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.0 MiB/s wr, 391 op/s; 388 KiB/s, 1 objects/s recovering 2024-03-26T21:06:47.431 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:47 smithi063 ceph-mon[66934]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-26T21:06:47.431 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:47 smithi063 ceph-mon[66934]: osdmap e128: 8 total, 7 up, 8 in 2024-03-26T21:06:47.484 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:47 smithi174 ceph-mon[53014]: pgmap v227: 97 pgs: 97 active+clean; 338 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.0 MiB/s wr, 391 op/s; 388 KiB/s, 1 objects/s recovering 2024-03-26T21:06:47.484 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:47 smithi174 ceph-mon[53014]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-26T21:06:47.484 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:47 smithi174 ceph-mon[53014]: osdmap e128: 8 total, 7 up, 8 in 2024-03-26T21:06:47.485 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:06:47 smithi174 podman[59708]: 2024-03-26 21:06:47.180903756 +0000 UTC m=+2.056484799 container cleanup 378c2aca7e71b7662f4b2639a030b3313e92e4d4bfef7b701d20daa65b470db8 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-5, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., ceph=True, release=754, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, GIT_BRANCH=HEAD, version=8, io.openshift.expose-services=, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, GIT_CLEAN=True, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-03-26T21:06:47.485 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:06:47 smithi174 bash[59708]: ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-5 2024-03-26T21:06:47.528 INFO:teuthology.orchestra.run.smithi063.stdout: 5 264260 10.01 MB/sec execute 521 sec latency 1065.127 ms 2024-03-26T21:06:47.798 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:06:47 smithi174 podman[59725]: 2024-03-26 21:06:47.491128018 +0000 UTC m=+0.675963987 container remove 378c2aca7e71b7662f4b2639a030b3313e92e4d4bfef7b701d20daa65b470db8 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-5, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, ceph=True, GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8) 2024-03-26T21:06:48.146 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:06:47 smithi174 podman[59816]: 2024-03-26 21:06:47.762410515 +0000 UTC m=+0.022058077 image pull a9bdc3eaeb106993bcbbac35d7fe5c7b76c052021676e129f5984aed30aee063 quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d 2024-03-26T21:06:48.147 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:06:47 smithi174 podman[59816]: 2024-03-26 21:06:47.908947301 +0000 UTC m=+0.168594834 container create 06db53eb95a5006f9cd9cb2d535c4ec6a83910dfcbf9c4255d8e4859922da77f (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-5-deactivate, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240326, ceph=True, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, io.buildah.version=1.33.5, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2) 2024-03-26T21:06:48.147 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:06:48 smithi174 podman[59816]: 2024-03-26 21:06:48.153279565 +0000 UTC m=+0.412927101 container init 06db53eb95a5006f9cd9cb2d535c4ec6a83910dfcbf9c4255d8e4859922da77f (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-5-deactivate, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-26T21:06:48.195 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:48 smithi063 ceph-mon[66934]: osdmap e129: 8 total, 7 up, 8 in 2024-03-26T21:06:48.503 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:48 smithi174 ceph-mon[53014]: osdmap e129: 8 total, 7 up, 8 in 2024-03-26T21:06:48.503 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:06:48 smithi174 podman[59816]: 2024-03-26 21:06:48.156857449 +0000 UTC m=+0.416504992 container start 06db53eb95a5006f9cd9cb2d535c4ec6a83910dfcbf9c4255d8e4859922da77f (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-5-deactivate, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240326, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-26T21:06:48.503 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:06:48 smithi174 podman[59816]: 2024-03-26 21:06:48.245035596 +0000 UTC m=+0.504683139 container attach 06db53eb95a5006f9cd9cb2d535c4ec6a83910dfcbf9c4255d8e4859922da77f (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-5-deactivate, org.label-schema.license=GPLv2, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, io.buildah.version=1.33.5, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b) 2024-03-26T21:06:48.528 INFO:teuthology.orchestra.run.smithi063.stdout: 5 264539 10.01 MB/sec execute 522 sec latency 43.865 ms 2024-03-26T21:06:48.798 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:06:48 smithi174 podman[59816]: 2024-03-26 21:06:48.510374577 +0000 UTC m=+0.770022104 container died 06db53eb95a5006f9cd9cb2d535c4ec6a83910dfcbf9c4255d8e4859922da77f (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-5-deactivate, GIT_BRANCH=HEAD, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=) 2024-03-26T21:06:49.223 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:49 smithi063 ceph-mon[66934]: pgmap v230: 97 pgs: 15 peering, 82 active+clean; 332 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.1 MiB/s wr, 484 op/s 2024-03-26T21:06:49.223 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:49 smithi063 ceph-mon[66934]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-03-26T21:06:49.471 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:49 smithi174 ceph-mon[53014]: pgmap v230: 97 pgs: 15 peering, 82 active+clean; 332 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.1 MiB/s wr, 484 op/s 2024-03-26T21:06:49.472 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:49 smithi174 ceph-mon[53014]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-03-26T21:06:49.472 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:06:49 smithi174 podman[59841]: 2024-03-26 21:06:49.22475736 +0000 UTC m=+0.709013257 container remove 06db53eb95a5006f9cd9cb2d535c4ec6a83910dfcbf9c4255d8e4859922da77f (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-5-deactivate, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, ceph=True) 2024-03-26T21:06:49.472 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:06:49 smithi174 systemd[1]: ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715@osd.5.service: Deactivated successfully. 2024-03-26T21:06:49.472 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:06:49 smithi174 systemd[1]: Stopped Ceph osd.5 for 0ed4dd4c-ebb2-11ee-95d0-87774f69a715. 2024-03-26T21:06:49.472 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:06:49 smithi174 systemd[1]: ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715@osd.5.service: Consumed 5min 34.182s CPU time. 2024-03-26T21:06:49.527 INFO:teuthology.orchestra.run.smithi063.stdout: 5 264928 10.02 MB/sec execute 523 sec latency 48.476 ms 2024-03-26T21:06:49.736 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:06:49 smithi174 systemd[1]: Starting Ceph osd.5 for 0ed4dd4c-ebb2-11ee-95d0-87774f69a715... 2024-03-26T21:06:49.736 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:06:49 smithi174 podman[59938]: 2024-03-26 21:06:49.625921038 +0000 UTC m=+0.020844059 image pull a9bdc3eaeb106993bcbbac35d7fe5c7b76c052021676e129f5984aed30aee063 quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d 2024-03-26T21:06:49.989 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:06:49 smithi174 podman[59938]: 2024-03-26 21:06:49.742534005 +0000 UTC m=+0.137456999 container create 242a912728acd41fb0cd0a6ed82d283813575deb78a113d7301b90c71a0cb499 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-5-activate, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD) 2024-03-26T21:06:49.989 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:06:49 smithi174 podman[59938]: 2024-03-26 21:06:49.987229824 +0000 UTC m=+0.382152830 container init 242a912728acd41fb0cd0a6ed82d283813575deb78a113d7301b90c71a0cb499 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-5-activate, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240326, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-26T21:06:50.247 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:50 smithi063 ceph-mon[66934]: Health check failed: Degraded data redundancy: 39/1575 objects degraded (2.476%), 6 pgs degraded (PG_DEGRADED) 2024-03-26T21:06:50.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:50 smithi174 ceph-mon[53014]: Health check failed: Degraded data redundancy: 39/1575 objects degraded (2.476%), 6 pgs degraded (PG_DEGRADED) 2024-03-26T21:06:50.298 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:06:49 smithi174 podman[59938]: 2024-03-26 21:06:49.996684438 +0000 UTC m=+0.391607442 container start 242a912728acd41fb0cd0a6ed82d283813575deb78a113d7301b90c71a0cb499 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-5-activate, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, GIT_BRANCH=HEAD, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.33.5, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-26T21:06:50.299 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:06:50 smithi174 podman[59938]: 2024-03-26 21:06:50.087108244 +0000 UTC m=+0.482031246 container attach 242a912728acd41fb0cd0a6ed82d283813575deb78a113d7301b90c71a0cb499 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-5-activate, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240326, ceph=True, CEPH_POINT_RELEASE=) 2024-03-26T21:06:50.528 INFO:teuthology.orchestra.run.smithi063.stdout: 5 265350 10.02 MB/sec execute 524 sec latency 57.317 ms 2024-03-26T21:06:50.920 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:06:50 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-5-activate[59953]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-03-26T21:06:50.920 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:06:50 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-5-activate[59953]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-03-26T21:06:50.920 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:06:50 smithi174 bash[59938]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-03-26T21:06:50.920 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:06:50 smithi174 bash[59938]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-03-26T21:06:50.921 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:06:50 smithi174 bash[59938]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-03-26T21:06:50.921 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:06:50 smithi174 bash[59938]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-03-26T21:06:50.921 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:06:50 smithi174 bash[59938]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-03-26T21:06:50.921 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:06:50 smithi174 bash[59938]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-03-26T21:06:50.921 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:06:50 smithi174 bash[59938]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-03-26T21:06:50.921 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:06:50 smithi174 bash[59938]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-03-26T21:06:50.921 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:06:50 smithi174 bash[59938]: --> ceph-volume raw activate successful for osd ID: 5 2024-03-26T21:06:50.921 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:06:50 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-5-activate[59953]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-03-26T21:06:50.921 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:06:50 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-5-activate[59953]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-03-26T21:06:50.921 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:06:50 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-5-activate[59953]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-03-26T21:06:50.921 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:06:50 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-5-activate[59953]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-03-26T21:06:50.921 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:06:50 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-5-activate[59953]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-03-26T21:06:50.921 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:06:50 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-5-activate[59953]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-03-26T21:06:50.921 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:06:50 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-5-activate[59953]: --> ceph-volume raw activate successful for osd ID: 5 2024-03-26T21:06:50.922 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:06:50 smithi174 podman[59938]: 2024-03-26 21:06:50.609775734 +0000 UTC m=+1.004698743 container died 242a912728acd41fb0cd0a6ed82d283813575deb78a113d7301b90c71a0cb499 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-5-activate, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240326, GIT_CLEAN=True, maintainer=Guillaume Abrioux , RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-26T21:06:51.275 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:51 smithi063 ceph-mon[66934]: pgmap v231: 97 pgs: 4 active+undersized, 6 active+undersized+degraded, 15 peering, 72 active+clean; 336 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.4 KiB/s rd, 4.0 MiB/s wr, 425 op/s; 39/1575 objects degraded (2.476%); 511 B/s, 0 objects/s recovering 2024-03-26T21:06:51.491 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:51 smithi174 ceph-mon[53014]: pgmap v231: 97 pgs: 4 active+undersized, 6 active+undersized+degraded, 15 peering, 72 active+clean; 336 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.4 KiB/s rd, 4.0 MiB/s wr, 425 op/s; 39/1575 objects degraded (2.476%); 511 B/s, 0 objects/s recovering 2024-03-26T21:06:51.492 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:06:51 smithi174 podman[60086]: 2024-03-26 21:06:51.270843576 +0000 UTC m=+0.655110695 container remove 242a912728acd41fb0cd0a6ed82d283813575deb78a113d7301b90c71a0cb499 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-5-activate, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-03-26T21:06:51.527 INFO:teuthology.orchestra.run.smithi063.stdout: 5 265686 10.03 MB/sec execute 525 sec latency 42.703 ms 2024-03-26T21:06:51.789 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:06:51 smithi174 podman[60143]: 2024-03-26 21:06:51.402213619 +0000 UTC m=+0.021966230 image pull a9bdc3eaeb106993bcbbac35d7fe5c7b76c052021676e129f5984aed30aee063 quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d 2024-03-26T21:06:51.789 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:06:51 smithi174 podman[60143]: 2024-03-26 21:06:51.543191613 +0000 UTC m=+0.162944250 container create 25dde01e0676876c4bf2abbdea177dabaeca8edb525a42040e9b1f434af7e9cd (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-5, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b) 2024-03-26T21:06:52.048 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:06:51 smithi174 podman[60143]: 2024-03-26 21:06:51.796216895 +0000 UTC m=+0.415969496 container init 25dde01e0676876c4bf2abbdea177dabaeca8edb525a42040e9b1f434af7e9cd (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-5, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-03-26T21:06:52.048 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:06:51 smithi174 podman[60143]: 2024-03-26 21:06:51.800333489 +0000 UTC m=+0.420086090 container start 25dde01e0676876c4bf2abbdea177dabaeca8edb525a42040e9b1f434af7e9cd (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-5, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240326) 2024-03-26T21:06:52.048 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:06:51 smithi174 ceph-osd[60161]: -- 172.21.15.174:0/1509274366 <== mon.1 v2:172.21.15.174:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5626375cd680 con 0x562637597000 2024-03-26T21:06:52.048 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:06:51 smithi174 bash[60143]: 25dde01e0676876c4bf2abbdea177dabaeca8edb525a42040e9b1f434af7e9cd 2024-03-26T21:06:52.048 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:06:51 smithi174 systemd[1]: Started Ceph osd.5 for 0ed4dd4c-ebb2-11ee-95d0-87774f69a715. 2024-03-26T21:06:52.528 INFO:teuthology.orchestra.run.smithi063.stdout: 5 265937 10.02 MB/sec execute 526 sec latency 42.151 ms 2024-03-26T21:06:52.798 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:06:52 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-5[60157]: 2024-03-26T21:06:52.343+0000 7f28247067c0 -1 Falling back to public interface 2024-03-26T21:06:53.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:52 smithi063 ceph-mon[66934]: pgmap v232: 97 pgs: 7 active+undersized, 21 active+undersized+degraded, 3 peering, 66 active+clean; 335 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.1 MiB/s wr, 564 op/s; 160/1590 objects degraded (10.063%); 511 B/s, 0 objects/s recovering 2024-03-26T21:06:53.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:52 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:06:53.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:52 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:06:53.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:52 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:06:53.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:52 smithi063 ceph-mon[66934]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2024-03-26T21:06:53.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:52 smithi174 ceph-mon[53014]: pgmap v232: 97 pgs: 7 active+undersized, 21 active+undersized+degraded, 3 peering, 66 active+clean; 335 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.1 MiB/s wr, 564 op/s; 160/1590 objects degraded (10.063%); 511 B/s, 0 objects/s recovering 2024-03-26T21:06:53.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:52 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:06:53.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:52 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:06:53.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:52 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:06:53.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:52 smithi174 ceph-mon[53014]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2024-03-26T21:06:53.528 INFO:teuthology.orchestra.run.smithi063.stdout: 5 266236 10.02 MB/sec execute 527 sec latency 55.812 ms 2024-03-26T21:06:54.528 INFO:teuthology.orchestra.run.smithi063.stdout: 5 266548 10.01 MB/sec execute 528 sec latency 41.968 ms 2024-03-26T21:06:55.086 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:54 smithi063 ceph-mon[66934]: pgmap v233: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.1 MiB/s wr, 552 op/s; 183/1602 objects degraded (11.423%); 511 B/s, 0 objects/s recovering 2024-03-26T21:06:55.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:54 smithi174 ceph-mon[53014]: pgmap v233: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.1 MiB/s wr, 552 op/s; 183/1602 objects degraded (11.423%); 511 B/s, 0 objects/s recovering 2024-03-26T21:06:55.528 INFO:teuthology.orchestra.run.smithi063.stdout: 5 266782 10.01 MB/sec execute 529 sec latency 51.709 ms 2024-03-26T21:06:56.528 INFO:teuthology.orchestra.run.smithi063.stdout: 5 267016 10.00 MB/sec execute 530 sec latency 78.795 ms 2024-03-26T21:06:56.876 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:56 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:06:56.876 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:56 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:06:56.876 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:56 smithi063 ceph-mon[66934]: pgmap v234: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 342 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 5.1 MiB/s wr, 544 op/s; 196/1692 objects degraded (11.584%); 427 B/s, 0 objects/s recovering 2024-03-26T21:06:56.876 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:56 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:06:56.876 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:56 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:06:57.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:56 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:06:57.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:56 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:06:57.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:56 smithi174 ceph-mon[53014]: pgmap v234: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 342 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 5.1 MiB/s wr, 544 op/s; 196/1692 objects degraded (11.584%); 427 B/s, 0 objects/s recovering 2024-03-26T21:06:57.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:56 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:06:57.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:56 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:06:57.528 INFO:teuthology.orchestra.run.smithi063.stdout: 5 267352 10.00 MB/sec execute 531 sec latency 69.400 ms 2024-03-26T21:06:57.889 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:57 smithi063 ceph-mon[66934]: Health check update: Degraded data redundancy: 196/1692 objects degraded (11.584%), 23 pgs degraded (PG_DEGRADED) 2024-03-26T21:06:57.889 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:57 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:06:57.889 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:57 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:06:57.956 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:57 smithi174 ceph-mon[53014]: Health check update: Degraded data redundancy: 196/1692 objects degraded (11.584%), 23 pgs degraded (PG_DEGRADED) 2024-03-26T21:06:57.957 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:57 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:06:57.957 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:57 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:06:58.528 INFO:teuthology.orchestra.run.smithi063.stdout: 5 267735 9.98 MB/sec execute 532 sec latency 57.607 ms 2024-03-26T21:06:58.912 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:06:58 smithi063 ceph-mon[66934]: pgmap v235: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 338 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.4 MiB/s wr, 422 op/s; 190/1614 objects degraded (11.772%); 387 B/s, 0 objects/s recovering 2024-03-26T21:06:59.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:06:58 smithi174 ceph-mon[53014]: pgmap v235: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 338 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.4 MiB/s wr, 422 op/s; 190/1614 objects degraded (11.772%); 387 B/s, 0 objects/s recovering 2024-03-26T21:06:59.528 INFO:teuthology.orchestra.run.smithi063.stdout: 5 268031 9.98 MB/sec execute 533 sec latency 55.407 ms 2024-03-26T21:07:00.528 INFO:teuthology.orchestra.run.smithi063.stdout: 5 268459 10.00 MB/sec execute 534 sec latency 41.158 ms 2024-03-26T21:07:00.932 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:00 smithi063 ceph-mon[66934]: pgmap v236: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 334 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.2 MiB/s wr, 424 op/s; 174/1497 objects degraded (11.623%); 341 B/s, 0 objects/s recovering 2024-03-26T21:07:01.048 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:07:00 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-5[60157]: 2024-03-26T21:07:00.747+0000 7f28247067c0 -1 osd.5 0 read_superblock omap replica is missing. 2024-03-26T21:07:01.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:00 smithi174 ceph-mon[53014]: pgmap v236: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 334 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.2 MiB/s wr, 424 op/s; 174/1497 objects degraded (11.623%); 341 B/s, 0 objects/s recovering 2024-03-26T21:07:01.528 INFO:teuthology.orchestra.run.smithi063.stdout: 5 268944 10.00 MB/sec execute 535 sec latency 46.720 ms 2024-03-26T21:07:02.528 INFO:teuthology.orchestra.run.smithi063.stdout: 5 269275 10.00 MB/sec execute 536 sec latency 29.696 ms 2024-03-26T21:07:02.548 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:07:02 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-5[60157]: 2024-03-26T21:07:02.082+0000 7f28247067c0 -1 osd.5 127 log_to_monitors true 2024-03-26T21:07:02.961 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:02 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:07:02.961 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:02 smithi063 ceph-mon[66934]: pgmap v237: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 330 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 4.6 MiB/s wr, 448 op/s; 192/1563 objects degraded (12.284%) 2024-03-26T21:07:02.961 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:02 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:07:02.961 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:02 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:07:02.962 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:02 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T21:07:02.962 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:02 smithi063 ceph-mon[66934]: pgmap v238: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 330 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.2 MiB/s wr, 410 op/s; 192/1563 objects degraded (12.284%) 2024-03-26T21:07:02.962 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:02 smithi063 ceph-mon[66934]: pgmap v239: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 330 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 4.7 MiB/s wr, 493 op/s; 192/1563 objects degraded (12.284%) 2024-03-26T21:07:02.962 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:02 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:07:02.962 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:02 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:07:02.962 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:02 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:07:02.962 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:02 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:07:02.962 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:02 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:07:02.962 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:02 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:07:02.962 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:02 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-03-26T21:07:02.962 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:02 smithi063 ceph-mon[66934]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-03-26T21:07:02.962 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:02 smithi063 ceph-mon[66934]: Upgrade: unsafe to stop osd(s) at this time (13 PGs are or would become offline) 2024-03-26T21:07:02.962 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:02 smithi063 ceph-mon[66934]: from='osd.5 [v2:172.21.15.174:6808/2294591808,v1:172.21.15.174:6809/2294591808]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-03-26T21:07:02.962 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:02 smithi063 ceph-mon[66934]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-03-26T21:07:03.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:02 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:07:03.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:02 smithi174 ceph-mon[53014]: pgmap v237: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 330 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 4.6 MiB/s wr, 448 op/s; 192/1563 objects degraded (12.284%) 2024-03-26T21:07:03.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:02 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:07:03.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:02 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:07:03.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:02 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T21:07:03.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:02 smithi174 ceph-mon[53014]: pgmap v238: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 330 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.2 MiB/s wr, 410 op/s; 192/1563 objects degraded (12.284%) 2024-03-26T21:07:03.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:02 smithi174 ceph-mon[53014]: pgmap v239: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 330 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 4.7 MiB/s wr, 493 op/s; 192/1563 objects degraded (12.284%) 2024-03-26T21:07:03.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:02 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:07:03.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:02 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:07:03.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:02 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:07:03.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:02 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:07:03.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:02 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:07:03.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:02 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:07:03.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:02 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-03-26T21:07:03.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:02 smithi174 ceph-mon[53014]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-03-26T21:07:03.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:02 smithi174 ceph-mon[53014]: Upgrade: unsafe to stop osd(s) at this time (13 PGs are or would become offline) 2024-03-26T21:07:03.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:02 smithi174 ceph-mon[53014]: from='osd.5 [v2:172.21.15.174:6808/2294591808,v1:172.21.15.174:6809/2294591808]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-03-26T21:07:03.050 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:02 smithi174 ceph-mon[53014]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-03-26T21:07:03.528 INFO:teuthology.orchestra.run.smithi063.stdout: 5 269529 10.00 MB/sec execute 537 sec latency 110.326 ms 2024-03-26T21:07:03.548 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:07:03 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-5[60157]: 2024-03-26T21:07:03.249+0000 7f2811b56700 -1 osd.5 127 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-26T21:07:03.861 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:03 smithi063 ceph-mon[66934]: Health check update: Degraded data redundancy: 192/1563 objects degraded (12.284%), 23 pgs degraded (PG_DEGRADED) 2024-03-26T21:07:03.861 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:03 smithi063 ceph-mon[66934]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-03-26T21:07:03.861 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:03 smithi063 ceph-mon[66934]: from='osd.5 [v2:172.21.15.174:6808/2294591808,v1:172.21.15.174:6809/2294591808]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]: dispatch 2024-03-26T21:07:03.861 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:03 smithi063 ceph-mon[66934]: osdmap e130: 8 total, 7 up, 8 in 2024-03-26T21:07:03.861 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:03 smithi063 ceph-mon[66934]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]: dispatch 2024-03-26T21:07:03.861 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:03 smithi063 ceph-mon[66934]: from='osd.5 ' entity='osd.5' 2024-03-26T21:07:04.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:03 smithi174 ceph-mon[53014]: Health check update: Degraded data redundancy: 192/1563 objects degraded (12.284%), 23 pgs degraded (PG_DEGRADED) 2024-03-26T21:07:04.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:03 smithi174 ceph-mon[53014]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-03-26T21:07:04.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:03 smithi174 ceph-mon[53014]: from='osd.5 [v2:172.21.15.174:6808/2294591808,v1:172.21.15.174:6809/2294591808]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]: dispatch 2024-03-26T21:07:04.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:03 smithi174 ceph-mon[53014]: osdmap e130: 8 total, 7 up, 8 in 2024-03-26T21:07:04.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:03 smithi174 ceph-mon[53014]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]: dispatch 2024-03-26T21:07:04.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:03 smithi174 ceph-mon[53014]: from='osd.5 ' entity='osd.5' 2024-03-26T21:07:04.528 INFO:teuthology.orchestra.run.smithi063.stdout: 5 269838 9.99 MB/sec execute 538 sec latency 214.119 ms 2024-03-26T21:07:04.890 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:04 smithi063 ceph-mon[66934]: pgmap v241: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 331 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.6 MiB/s wr, 406 op/s; 190/1581 objects degraded (12.018%) 2024-03-26T21:07:04.890 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:04 smithi063 ceph-mon[66934]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-26T21:07:04.890 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:04 smithi063 ceph-mon[66934]: osd.5 [v2:172.21.15.174:6808/2294591808,v1:172.21.15.174:6809/2294591808] boot 2024-03-26T21:07:04.890 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:04 smithi063 ceph-mon[66934]: osdmap e131: 8 total, 8 up, 8 in 2024-03-26T21:07:04.890 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:04 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-26T21:07:04.890 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:04 smithi063 ceph-mon[66934]: osdmap e132: 8 total, 8 up, 8 in 2024-03-26T21:07:05.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:04 smithi174 ceph-mon[53014]: pgmap v241: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 331 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.6 MiB/s wr, 406 op/s; 190/1581 objects degraded (12.018%) 2024-03-26T21:07:05.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:04 smithi174 ceph-mon[53014]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-26T21:07:05.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:04 smithi174 ceph-mon[53014]: osd.5 [v2:172.21.15.174:6808/2294591808,v1:172.21.15.174:6809/2294591808] boot 2024-03-26T21:07:05.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:04 smithi174 ceph-mon[53014]: osdmap e131: 8 total, 8 up, 8 in 2024-03-26T21:07:05.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:04 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-26T21:07:05.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:04 smithi174 ceph-mon[53014]: osdmap e132: 8 total, 8 up, 8 in 2024-03-26T21:07:05.529 INFO:teuthology.orchestra.run.smithi063.stdout: 5 270042 9.98 MB/sec execute 539 sec latency 1215.261 ms 2024-03-26T21:07:06.529 INFO:teuthology.orchestra.run.smithi063.stdout: 5 270307 9.98 MB/sec execute 540 sec latency 1365.112 ms 2024-03-26T21:07:06.728 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:06 smithi063 ceph-mon[66934]: osdmap e133: 8 total, 8 up, 8 in 2024-03-26T21:07:06.729 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:06 smithi063 ceph-mon[66934]: pgmap v245: 97 pgs: 2 activating+degraded, 2 remapped+peering, 6 active+undersized, 19 active+undersized+degraded, 68 active+clean; 339 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.9 MiB/s wr, 377 op/s; 172/1638 objects degraded (10.501%) 2024-03-26T21:07:07.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:06 smithi174 ceph-mon[53014]: osdmap e133: 8 total, 8 up, 8 in 2024-03-26T21:07:07.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:06 smithi174 ceph-mon[53014]: pgmap v245: 97 pgs: 2 activating+degraded, 2 remapped+peering, 6 active+undersized, 19 active+undersized+degraded, 68 active+clean; 339 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.9 MiB/s wr, 377 op/s; 172/1638 objects degraded (10.501%) 2024-03-26T21:07:07.183 INFO:teuthology.orchestra.run.smithi063.stdout:true 2024-03-26T21:07:07.529 INFO:teuthology.orchestra.run.smithi063.stdout: 5 270577 9.98 MB/sec execute 541 sec latency 896.413 ms 2024-03-26T21:07:07.753 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:07 smithi063 ceph-mon[66934]: osdmap e134: 8 total, 8 up, 8 in 2024-03-26T21:07:07.754 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:07 smithi063 ceph-mon[66934]: from='client.34388 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:07:08.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:07 smithi174 ceph-mon[53014]: osdmap e134: 8 total, 8 up, 8 in 2024-03-26T21:07:08.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:07 smithi174 ceph-mon[53014]: from='client.34388 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:07:08.133 INFO:teuthology.orchestra.run.smithi063.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-26T21:07:08.133 INFO:teuthology.orchestra.run.smithi063.stdout:alertmanager.smithi063 smithi063 *:9093,9094 running (8m) 87s ago 17m 24.5M - 0.23.0 ba2b418f427c 6e160c94aac3 2024-03-26T21:07:08.133 INFO:teuthology.orchestra.run.smithi063.stdout:crash.smithi063 smithi063 running (4m) 87s ago 17m 6665k - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 cf1096d84270 2024-03-26T21:07:08.133 INFO:teuthology.orchestra.run.smithi063.stdout:crash.smithi174 smithi174 running (4m) 12s ago 16m 6665k - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 cb5371a92684 2024-03-26T21:07:08.133 INFO:teuthology.orchestra.run.smithi063.stdout:grafana.smithi063 smithi063 *:3000 error 87s ago 17m - - 2024-03-26T21:07:08.133 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foofs.smithi063.tzaznz smithi063 running (11m) 87s ago 11m 37.8M - 17.2.0 e1d6a67b021e 10a989fc7954 2024-03-26T21:07:08.133 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foofs.smithi174.gitdvk smithi174 running (11m) 12s ago 11m 14.2M - 17.2.0 e1d6a67b021e 4339a5d98d1d 2024-03-26T21:07:08.133 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.smithi063.lxhujt smithi063 *:8443,9283,8765 running (7m) 87s ago 18m 509M - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 6d9d10330a90 2024-03-26T21:07:08.133 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.smithi174.sexoin smithi174 *:8443,9283,8765 running (6m) 12s ago 16m 452M - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 d7546233672d 2024-03-26T21:07:08.134 INFO:teuthology.orchestra.run.smithi063.stdout:mon.smithi063 smithi063 running (6m) 87s ago 18m 52.3M 2048M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 c644200344f8 2024-03-26T21:07:08.134 INFO:teuthology.orchestra.run.smithi063.stdout:mon.smithi174 smithi174 running (5m) 12s ago 16m 45.7M 2048M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 1a6d442692e7 2024-03-26T21:07:08.134 INFO:teuthology.orchestra.run.smithi063.stdout:nfs.foo.0.0.smithi063.fgmpwa smithi063 *:2049 running (11m) 87s ago 11m 275M - 4.0 e1d6a67b021e 3d6c0dd9ce51 2024-03-26T21:07:08.134 INFO:teuthology.orchestra.run.smithi063.stdout:nfs.foo.1.0.smithi174.pytptt smithi174 *:2049 running (11m) 12s ago 11m 86.9M - 4.0 e1d6a67b021e f220dbb449a3 2024-03-26T21:07:08.134 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.smithi063 smithi063 *:9100 running (7m) 87s ago 17m 20.7M - 1.3.1 1dbe0e931976 f52f41fd14c0 2024-03-26T21:07:08.134 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.smithi174 smithi174 *:9100 running (7m) 12s ago 16m 19.2M - 1.3.1 1dbe0e931976 0c027f39a4c8 2024-03-26T21:07:08.134 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0 smithi063 running (3m) 87s ago 16m 339M 1936M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 2d4b3fa825cf 2024-03-26T21:07:08.134 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1 smithi063 running (2m) 87s ago 15m 282M 1936M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 d41fbe507197 2024-03-26T21:07:08.134 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2 smithi063 running (2m) 87s ago 15m 200M 1936M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 f123eeabd3fa 2024-03-26T21:07:08.134 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3 smithi063 running (98s) 87s ago 14m 15.3M 1936M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 953816eafe87 2024-03-26T21:07:08.134 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4 smithi174 running (54s) 12s ago 14m 273M 2704M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 8aa35bff6cea 2024-03-26T21:07:08.134 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5 smithi174 running (16s) 12s ago 13m 14.8M 2704M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 25dde01e0676 2024-03-26T21:07:08.134 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6 smithi174 running (13m) 12s ago 13m 561M 2704M 17.2.0 e1d6a67b021e ffbb5b7ad3b6 2024-03-26T21:07:08.134 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7 smithi174 running (12m) 12s ago 12m 584M 2704M 17.2.0 e1d6a67b021e c1d57fcabb3f 2024-03-26T21:07:08.135 INFO:teuthology.orchestra.run.smithi063.stdout:prometheus.smithi063 smithi063 *:9095 running (6m) 87s ago 16m 76.0M - 2.33.4 514e6a882f6e 567eef202627 2024-03-26T21:07:08.529 INFO:teuthology.orchestra.run.smithi063.stdout: 5 270883 9.96 MB/sec execute 542 sec latency 1042.784 ms 2024-03-26T21:07:08.670 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-03-26T21:07:08.670 INFO:teuthology.orchestra.run.smithi063.stdout: "mon": { 2024-03-26T21:07:08.670 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 2 2024-03-26T21:07:08.670 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:07:08.670 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr": { 2024-03-26T21:07:08.670 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 2 2024-03-26T21:07:08.670 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:07:08.670 INFO:teuthology.orchestra.run.smithi063.stdout: "osd": { 2024-03-26T21:07:08.670 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-03-26T21:07:08.670 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 6 2024-03-26T21:07:08.671 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:07:08.671 INFO:teuthology.orchestra.run.smithi063.stdout: "mds": { 2024-03-26T21:07:08.671 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-26T21:07:08.671 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:07:08.671 INFO:teuthology.orchestra.run.smithi063.stdout: "overall": { 2024-03-26T21:07:08.671 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-03-26T21:07:08.671 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 10 2024-03-26T21:07:08.671 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-03-26T21:07:08.671 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-03-26T21:07:08.767 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:08 smithi063 ceph-mon[66934]: osdmap e135: 8 total, 8 up, 8 in 2024-03-26T21:07:08.768 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:08 smithi063 ceph-mon[66934]: from='client.34392 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:07:08.768 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:08 smithi063 ceph-mon[66934]: pgmap v248: 97 pgs: 2 peering, 2 activating+degraded, 2 remapped+peering, 4 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 76 active+clean; 343 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 7.1 MiB/s wr, 728 op/s; 114/1608 objects degraded (7.090%); 1.6 MiB/s, 20 objects/s recovering 2024-03-26T21:07:08.768 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:08 smithi063 ceph-mon[66934]: from='client.34396 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:07:08.768 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:08 smithi063 ceph-mon[66934]: osdmap e136: 8 total, 8 up, 8 in 2024-03-26T21:07:09.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:08 smithi174 ceph-mon[53014]: osdmap e135: 8 total, 8 up, 8 in 2024-03-26T21:07:09.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:08 smithi174 ceph-mon[53014]: from='client.34392 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:07:09.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:08 smithi174 ceph-mon[53014]: pgmap v248: 97 pgs: 2 peering, 2 activating+degraded, 2 remapped+peering, 4 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 76 active+clean; 343 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 7.1 MiB/s wr, 728 op/s; 114/1608 objects degraded (7.090%); 1.6 MiB/s, 20 objects/s recovering 2024-03-26T21:07:09.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:08 smithi174 ceph-mon[53014]: from='client.34396 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:07:09.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:08 smithi174 ceph-mon[53014]: osdmap e136: 8 total, 8 up, 8 in 2024-03-26T21:07:09.146 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-03-26T21:07:09.146 INFO:teuthology.orchestra.run.smithi063.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d", 2024-03-26T21:07:09.146 INFO:teuthology.orchestra.run.smithi063.stdout: "in_progress": true, 2024-03-26T21:07:09.146 INFO:teuthology.orchestra.run.smithi063.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-26T21:07:09.146 INFO:teuthology.orchestra.run.smithi063.stdout: "services_complete": [ 2024-03-26T21:07:09.146 INFO:teuthology.orchestra.run.smithi063.stdout: "crash", 2024-03-26T21:07:09.147 INFO:teuthology.orchestra.run.smithi063.stdout: "mon", 2024-03-26T21:07:09.147 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr" 2024-03-26T21:07:09.147 INFO:teuthology.orchestra.run.smithi063.stdout: ], 2024-03-26T21:07:09.147 INFO:teuthology.orchestra.run.smithi063.stdout: "progress": "12/23 daemons upgraded", 2024-03-26T21:07:09.147 INFO:teuthology.orchestra.run.smithi063.stdout: "message": "Currently upgrading osd daemons", 2024-03-26T21:07:09.147 INFO:teuthology.orchestra.run.smithi063.stdout: "is_paused": false 2024-03-26T21:07:09.147 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-03-26T21:07:09.536 INFO:teuthology.orchestra.run.smithi063.stdout: 5 271190 9.96 MB/sec execute 543 sec latency 832.138 ms 2024-03-26T21:07:09.708 INFO:teuthology.orchestra.run.smithi063.stdout:HEALTH_WARN 1 failed cephadm daemon(s); Degraded data redundancy: 114/1608 objects degraded (7.090%), 13 pgs degraded 2024-03-26T21:07:09.709 INFO:teuthology.orchestra.run.smithi063.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-26T21:07:09.709 INFO:teuthology.orchestra.run.smithi063.stdout: daemon grafana.smithi063 on smithi063 is in error state 2024-03-26T21:07:09.709 INFO:teuthology.orchestra.run.smithi063.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 114/1608 objects degraded (7.090%), 13 pgs degraded 2024-03-26T21:07:09.709 INFO:teuthology.orchestra.run.smithi063.stdout: pg 2.2 is active+undersized+degraded, acting [1,6] 2024-03-26T21:07:09.709 INFO:teuthology.orchestra.run.smithi063.stdout: pg 2.3 is active+undersized+degraded, acting [2,4] 2024-03-26T21:07:09.709 INFO:teuthology.orchestra.run.smithi063.stdout: pg 2.12 is active+undersized+degraded, acting [3,7] 2024-03-26T21:07:09.710 INFO:teuthology.orchestra.run.smithi063.stdout: pg 2.15 is activating+degraded, acting [1,5,0] 2024-03-26T21:07:09.710 INFO:teuthology.orchestra.run.smithi063.stdout: pg 2.16 is active+undersized+degraded, acting [6,2] 2024-03-26T21:07:09.710 INFO:teuthology.orchestra.run.smithi063.stdout: pg 2.18 is active+undersized+degraded, acting [4,7] 2024-03-26T21:07:09.710 INFO:teuthology.orchestra.run.smithi063.stdout: pg 3.2 is active+recovery_wait+undersized+degraded+remapped, acting [3,6] 2024-03-26T21:07:09.710 INFO:teuthology.orchestra.run.smithi063.stdout: pg 3.4 is activating+degraded, acting [1,2,5] 2024-03-26T21:07:09.710 INFO:teuthology.orchestra.run.smithi063.stdout: pg 3.5 is active+undersized+degraded, acting [3,2] 2024-03-26T21:07:09.710 INFO:teuthology.orchestra.run.smithi063.stdout: pg 3.c is active+undersized+degraded, acting [3,6] 2024-03-26T21:07:09.711 INFO:teuthology.orchestra.run.smithi063.stdout: pg 3.d is active+recovering+undersized+degraded+remapped, acting [7,6] 2024-03-26T21:07:09.711 INFO:teuthology.orchestra.run.smithi063.stdout: pg 3.1c is active+undersized+degraded, acting [4,1] 2024-03-26T21:07:09.711 INFO:teuthology.orchestra.run.smithi063.stdout: pg 3.1d is active+undersized+degraded, acting [4,6] 2024-03-26T21:07:09.793 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:09 smithi063 ceph-mon[66934]: Health check update: Degraded data redundancy: 114/1608 objects degraded (7.090%), 13 pgs degraded (PG_DEGRADED) 2024-03-26T21:07:09.793 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:09 smithi063 ceph-mon[66934]: from='client.? 172.21.15.63:0/1710656505' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:07:09.793 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:09 smithi063 ceph-mon[66934]: from='client.34404 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:07:10.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:09 smithi174 ceph-mon[53014]: Health check update: Degraded data redundancy: 114/1608 objects degraded (7.090%), 13 pgs degraded (PG_DEGRADED) 2024-03-26T21:07:10.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:09 smithi174 ceph-mon[53014]: from='client.? 172.21.15.63:0/1710656505' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:07:10.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:09 smithi174 ceph-mon[53014]: from='client.34404 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:07:10.536 INFO:teuthology.orchestra.run.smithi063.stdout: 5 271466 9.95 MB/sec execute 544 sec latency 47.148 ms 2024-03-26T21:07:10.827 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:10 smithi063 ceph-mon[66934]: from='client.? 172.21.15.63:0/2347668219' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-26T21:07:10.827 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:10 smithi063 ceph-mon[66934]: pgmap v250: 97 pgs: 3 peering, 1 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 331 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.0 MiB/s wr, 392 op/s; 27/1560 objects degraded (1.731%); 1.7 MiB/s, 48 objects/s recovering 2024-03-26T21:07:10.827 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:10 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:07:10.827 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:10 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:07:11.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:10 smithi174 ceph-mon[53014]: from='client.? 172.21.15.63:0/2347668219' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-26T21:07:11.098 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:10 smithi174 ceph-mon[53014]: pgmap v250: 97 pgs: 3 peering, 1 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 331 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.0 MiB/s wr, 392 op/s; 27/1560 objects degraded (1.731%); 1.7 MiB/s, 48 objects/s recovering 2024-03-26T21:07:11.098 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:10 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:07:11.098 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:10 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:07:11.536 INFO:teuthology.orchestra.run.smithi063.stdout: 5 271956 9.96 MB/sec execute 545 sec latency 34.697 ms 2024-03-26T21:07:12.086 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:11 smithi063 ceph-mon[66934]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 27/1560 objects degraded (1.731%), 2 pgs degraded) 2024-03-26T21:07:12.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:11 smithi174 ceph-mon[53014]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 27/1560 objects degraded (1.731%), 2 pgs degraded) 2024-03-26T21:07:12.536 INFO:teuthology.orchestra.run.smithi063.stdout: 5 272567 9.98 MB/sec execute 546 sec latency 29.312 ms 2024-03-26T21:07:13.096 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:12 smithi063 ceph-mon[66934]: pgmap v251: 97 pgs: 1 peering, 96 active+clean; 335 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 5.4 MiB/s wr, 533 op/s; 2.1 MiB/s, 39 objects/s recovering 2024-03-26T21:07:13.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:12 smithi174 ceph-mon[53014]: pgmap v251: 97 pgs: 1 peering, 96 active+clean; 335 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 5.4 MiB/s wr, 533 op/s; 2.1 MiB/s, 39 objects/s recovering 2024-03-26T21:07:13.536 INFO:teuthology.orchestra.run.smithi063.stdout: 5 272894 9.98 MB/sec execute 547 sec latency 40.622 ms 2024-03-26T21:07:14.536 INFO:teuthology.orchestra.run.smithi063.stdout: 5 273240 9.98 MB/sec execute 548 sec latency 42.264 ms 2024-03-26T21:07:14.870 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:14 smithi063 ceph-mon[66934]: pgmap v252: 97 pgs: 97 active+clean; 331 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 9.4 KiB/s rd, 4.1 MiB/s wr, 349 op/s; 1.7 MiB/s, 33 objects/s recovering 2024-03-26T21:07:15.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:14 smithi174 ceph-mon[53014]: pgmap v252: 97 pgs: 97 active+clean; 331 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 9.4 KiB/s rd, 4.1 MiB/s wr, 349 op/s; 1.7 MiB/s, 33 objects/s recovering 2024-03-26T21:07:15.536 INFO:teuthology.orchestra.run.smithi063.stdout: 5 273565 9.98 MB/sec execute 549 sec latency 53.523 ms 2024-03-26T21:07:16.537 INFO:teuthology.orchestra.run.smithi063.stdout: 5 273846 9.97 MB/sec execute 550 sec latency 42.892 ms 2024-03-26T21:07:16.927 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:16 smithi063 ceph-mon[66934]: pgmap v253: 97 pgs: 97 active+clean; 338 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4.4 KiB/s rd, 3.8 MiB/s wr, 438 op/s; 867 KiB/s, 20 objects/s recovering 2024-03-26T21:07:17.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:16 smithi174 ceph-mon[53014]: pgmap v253: 97 pgs: 97 active+clean; 338 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4.4 KiB/s rd, 3.8 MiB/s wr, 438 op/s; 867 KiB/s, 20 objects/s recovering 2024-03-26T21:07:17.536 INFO:teuthology.orchestra.run.smithi063.stdout: 5 274159 9.97 MB/sec execute 551 sec latency 45.028 ms 2024-03-26T21:07:17.932 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:17 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-03-26T21:07:17.932 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:17 smithi063 ceph-mon[66934]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-03-26T21:07:17.932 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:17 smithi063 ceph-mon[66934]: Upgrade: osd.6 is safe to restart 2024-03-26T21:07:18.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:17 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-03-26T21:07:18.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:17 smithi174 ceph-mon[53014]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-03-26T21:07:18.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:17 smithi174 ceph-mon[53014]: Upgrade: osd.6 is safe to restart 2024-03-26T21:07:18.536 INFO:teuthology.orchestra.run.smithi063.stdout: 5 274573 9.96 MB/sec execute 552 sec latency 51.863 ms 2024-03-26T21:07:18.959 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:18 smithi063 ceph-mon[66934]: pgmap v254: 97 pgs: 97 active+clean; 341 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.2 MiB/s wr, 521 op/s; 703 KiB/s, 16 objects/s recovering 2024-03-26T21:07:18.959 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:18 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:07:18.959 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:18 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-03-26T21:07:18.959 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:18 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:07:19.050 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:18 smithi174 ceph-mon[53014]: pgmap v254: 97 pgs: 97 active+clean; 341 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.2 MiB/s wr, 521 op/s; 703 KiB/s, 16 objects/s recovering 2024-03-26T21:07:19.050 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:18 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:07:19.050 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:18 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-03-26T21:07:19.050 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:18 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:07:19.536 INFO:teuthology.orchestra.run.smithi063.stdout: 5 274893 9.95 MB/sec execute 553 sec latency 59.269 ms 2024-03-26T21:07:19.993 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:19 smithi063 ceph-mon[66934]: Upgrade: Updating osd.6 2024-03-26T21:07:19.993 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:19 smithi063 ceph-mon[66934]: Deploying daemon osd.6 on smithi174 2024-03-26T21:07:20.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:19 smithi174 ceph-mon[53014]: Upgrade: Updating osd.6 2024-03-26T21:07:20.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:19 smithi174 ceph-mon[53014]: Deploying daemon osd.6 on smithi174 2024-03-26T21:07:20.536 INFO:teuthology.orchestra.run.smithi063.stdout: 5 275223 9.96 MB/sec execute 554 sec latency 45.407 ms 2024-03-26T21:07:21.024 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:20 smithi063 ceph-mon[66934]: pgmap v255: 97 pgs: 97 active+clean; 333 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.9 MiB/s wr, 488 op/s; 544 KiB/s, 4 objects/s recovering 2024-03-26T21:07:21.024 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:20 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:07:21.132 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:20 smithi174 ceph-mon[53014]: pgmap v255: 97 pgs: 97 active+clean; 333 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.9 MiB/s wr, 488 op/s; 544 KiB/s, 4 objects/s recovering 2024-03-26T21:07:21.132 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:20 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:07:21.536 INFO:teuthology.orchestra.run.smithi063.stdout: 5 275585 9.96 MB/sec execute 555 sec latency 56.871 ms 2024-03-26T21:07:22.537 INFO:teuthology.orchestra.run.smithi063.stdout: 5 276054 9.97 MB/sec execute 556 sec latency 37.125 ms 2024-03-26T21:07:22.883 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:07:22 smithi174 systemd[1]: Stopping Ceph osd.6 for 0ed4dd4c-ebb2-11ee-95d0-87774f69a715... 2024-03-26T21:07:23.083 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:22 smithi063 ceph-mon[66934]: pgmap v256: 97 pgs: 97 active+clean; 334 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 9.6 KiB/s rd, 5.6 MiB/s wr, 613 op/s; 434 KiB/s, 2 objects/s recovering 2024-03-26T21:07:23.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:22 smithi174 ceph-mon[53014]: pgmap v256: 97 pgs: 97 active+clean; 334 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 9.6 KiB/s rd, 5.6 MiB/s wr, 613 op/s; 434 KiB/s, 2 objects/s recovering 2024-03-26T21:07:23.298 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:07:22 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6[37195]: 2024-03-26T21:07:22.956+0000 7ff255116700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-26T21:07:23.298 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:07:22 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6[37195]: 2024-03-26T21:07:22.956+0000 7ff255116700 -1 osd.6 136 *** Got signal Terminated *** 2024-03-26T21:07:23.298 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:07:22 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6[37195]: 2024-03-26T21:07:22.956+0000 7ff255116700 -1 osd.6 136 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-26T21:07:23.536 INFO:teuthology.orchestra.run.smithi063.stdout: 5 276328 9.96 MB/sec execute 557 sec latency 38.466 ms 2024-03-26T21:07:24.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:23 smithi174 ceph-mon[53014]: osd.6 marked itself down and dead 2024-03-26T21:07:24.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:23 smithi063 ceph-mon[66934]: osd.6 marked itself down and dead 2024-03-26T21:07:24.537 INFO:teuthology.orchestra.run.smithi063.stdout: 5 276464 9.95 MB/sec execute 558 sec latency 659.896 ms 2024-03-26T21:07:24.798 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:07:24 smithi174 podman[62001]: 2024-03-26 21:07:24.545980703 +0000 UTC m=+1.732856852 container died ffbb5b7ad3b67266a3c2e40868d40a69b3c7523b207943cc7f5eede391014121 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6, name=centos-stream, GIT_BRANCH=HEAD, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, architecture=x86_64, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, release=754, distribution-scope=public, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870) 2024-03-26T21:07:25.139 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:24 smithi063 ceph-mon[66934]: pgmap v257: 97 pgs: 97 active+clean; 331 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 5.1 MiB/s wr, 510 op/s 2024-03-26T21:07:25.140 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:24 smithi063 ceph-mon[66934]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-26T21:07:25.140 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:24 smithi063 ceph-mon[66934]: osdmap e137: 8 total, 7 up, 8 in 2024-03-26T21:07:25.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:24 smithi174 ceph-mon[53014]: pgmap v257: 97 pgs: 97 active+clean; 331 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 5.1 MiB/s wr, 510 op/s 2024-03-26T21:07:25.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:24 smithi174 ceph-mon[53014]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-26T21:07:25.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:24 smithi174 ceph-mon[53014]: osdmap e137: 8 total, 7 up, 8 in 2024-03-26T21:07:25.299 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:07:25 smithi174 podman[62001]: 2024-03-26 21:07:25.007143402 +0000 UTC m=+2.194019553 container cleanup ffbb5b7ad3b67266a3c2e40868d40a69b3c7523b207943cc7f5eede391014121 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, ceph=True, architecture=x86_64, io.buildah.version=1.19.8, io.openshift.expose-services=, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, name=centos-stream, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-03-26T21:07:25.299 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:07:25 smithi174 bash[62001]: ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6 2024-03-26T21:07:25.537 INFO:teuthology.orchestra.run.smithi063.stdout: 5 276662 9.95 MB/sec execute 559 sec latency 1060.863 ms 2024-03-26T21:07:25.798 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:07:25 smithi174 podman[62016]: 2024-03-26 21:07:25.409478977 +0000 UTC m=+0.860367452 container remove ffbb5b7ad3b67266a3c2e40868d40a69b3c7523b207943cc7f5eede391014121 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, name=centos-stream, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, release=754, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.openshift.expose-services=, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-03-26T21:07:25.894 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:25 smithi063 ceph-mon[66934]: osdmap e138: 8 total, 7 up, 8 in 2024-03-26T21:07:25.894 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:25 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:07:25.894 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:25 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:07:26.106 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:25 smithi174 ceph-mon[53014]: osdmap e138: 8 total, 7 up, 8 in 2024-03-26T21:07:26.106 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:25 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:07:26.106 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:25 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:07:26.107 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:07:25 smithi174 podman[62100]: 2024-03-26 21:07:25.730187073 +0000 UTC m=+0.017938867 image pull a9bdc3eaeb106993bcbbac35d7fe5c7b76c052021676e129f5984aed30aee063 quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d 2024-03-26T21:07:26.107 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:07:25 smithi174 podman[62100]: 2024-03-26 21:07:25.852227264 +0000 UTC m=+0.139979075 container create c056ff74534cb9441d80cefcea6a65cf1a9eb3d5bfc6ab2c0bf8dce3e57cd6b8 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6-deactivate, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, maintainer=Guillaume Abrioux , GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2) 2024-03-26T21:07:26.107 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:07:26 smithi174 podman[62100]: 2024-03-26 21:07:26.113279234 +0000 UTC m=+0.401031011 container init c056ff74534cb9441d80cefcea6a65cf1a9eb3d5bfc6ab2c0bf8dce3e57cd6b8 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6-deactivate, maintainer=Guillaume Abrioux , GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240326, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5) 2024-03-26T21:07:26.474 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:07:26 smithi174 podman[62100]: 2024-03-26 21:07:26.117766022 +0000 UTC m=+0.405517793 container start c056ff74534cb9441d80cefcea6a65cf1a9eb3d5bfc6ab2c0bf8dce3e57cd6b8 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6-deactivate, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, ceph=True, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b) 2024-03-26T21:07:26.474 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:07:26 smithi174 podman[62100]: 2024-03-26 21:07:26.205051112 +0000 UTC m=+0.492802891 container attach c056ff74534cb9441d80cefcea6a65cf1a9eb3d5bfc6ab2c0bf8dce3e57cd6b8 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6-deactivate, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, GIT_CLEAN=True, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, org.label-schema.schema-version=1.0, ceph=True) 2024-03-26T21:07:26.475 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:07:26 smithi174 podman[62100]: 2024-03-26 21:07:26.481125904 +0000 UTC m=+0.768877688 container died c056ff74534cb9441d80cefcea6a65cf1a9eb3d5bfc6ab2c0bf8dce3e57cd6b8 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6-deactivate, GIT_BRANCH=HEAD, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5) 2024-03-26T21:07:26.537 INFO:teuthology.orchestra.run.smithi063.stdout: 5 277054 9.95 MB/sec execute 560 sec latency 34.115 ms 2024-03-26T21:07:26.913 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:26 smithi063 ceph-mon[66934]: pgmap v260: 97 pgs: 15 peering, 5 stale+active+clean, 77 active+clean; 332 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.6 MiB/s wr, 364 op/s 2024-03-26T21:07:26.913 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:26 smithi063 ceph-mon[66934]: Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2024-03-26T21:07:27.098 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:07:27 smithi174 podman[62125]: 2024-03-26 21:07:27.092919365 +0000 UTC m=+0.601189124 container remove c056ff74534cb9441d80cefcea6a65cf1a9eb3d5bfc6ab2c0bf8dce3e57cd6b8 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6-deactivate, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0) 2024-03-26T21:07:27.098 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:26 smithi174 ceph-mon[53014]: pgmap v260: 97 pgs: 15 peering, 5 stale+active+clean, 77 active+clean; 332 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.6 MiB/s wr, 364 op/s 2024-03-26T21:07:27.098 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:26 smithi174 ceph-mon[53014]: Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2024-03-26T21:07:27.537 INFO:teuthology.orchestra.run.smithi063.stdout: 5 277346 9.95 MB/sec execute 561 sec latency 48.538 ms 2024-03-26T21:07:27.548 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:07:27 smithi174 systemd[1]: ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715@osd.6.service: Deactivated successfully. 2024-03-26T21:07:27.548 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:07:27 smithi174 systemd[1]: Stopped Ceph osd.6 for 0ed4dd4c-ebb2-11ee-95d0-87774f69a715. 2024-03-26T21:07:27.548 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:07:27 smithi174 systemd[1]: ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715@osd.6.service: Consumed 5min 13.061s CPU time. 2024-03-26T21:07:27.548 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:07:27 smithi174 systemd[1]: Starting Ceph osd.6 for 0ed4dd4c-ebb2-11ee-95d0-87774f69a715... 2024-03-26T21:07:27.867 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:07:27 smithi174 podman[62225]: 2024-03-26 21:07:27.489442223 +0000 UTC m=+0.023030984 image pull a9bdc3eaeb106993bcbbac35d7fe5c7b76c052021676e129f5984aed30aee063 quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d 2024-03-26T21:07:27.867 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:07:27 smithi174 podman[62225]: 2024-03-26 21:07:27.619190311 +0000 UTC m=+0.152779051 container create 4fecd3a2fcba47948e692862717f85d6f347eda605a88c6b8ca2f3562fe6cbee (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5) 2024-03-26T21:07:27.956 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:27 smithi063 ceph-mon[66934]: Health check failed: Degraded data redundancy: 92/1683 objects degraded (5.466%), 11 pgs degraded (PG_DEGRADED) 2024-03-26T21:07:28.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:27 smithi174 ceph-mon[53014]: Health check failed: Degraded data redundancy: 92/1683 objects degraded (5.466%), 11 pgs degraded (PG_DEGRADED) 2024-03-26T21:07:28.298 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:07:27 smithi174 podman[62225]: 2024-03-26 21:07:27.873761111 +0000 UTC m=+0.407349866 container init 4fecd3a2fcba47948e692862717f85d6f347eda605a88c6b8ca2f3562fe6cbee (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6-activate, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-03-26T21:07:28.298 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:07:27 smithi174 podman[62225]: 2024-03-26 21:07:27.877233514 +0000 UTC m=+0.410822264 container start 4fecd3a2fcba47948e692862717f85d6f347eda605a88c6b8ca2f3562fe6cbee (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6-activate, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, ceph=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-03-26T21:07:28.298 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:07:27 smithi174 podman[62225]: 2024-03-26 21:07:27.972080245 +0000 UTC m=+0.505668990 container attach 4fecd3a2fcba47948e692862717f85d6f347eda605a88c6b8ca2f3562fe6cbee (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6-activate, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b) 2024-03-26T21:07:28.537 INFO:teuthology.orchestra.run.smithi063.stdout: 5 277667 9.95 MB/sec execute 562 sec latency 49.009 ms 2024-03-26T21:07:28.795 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:07:28 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6-activate[62239]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-03-26T21:07:28.796 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:07:28 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6-activate[62239]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-03-26T21:07:28.796 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:07:28 smithi174 bash[62225]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-03-26T21:07:28.796 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:07:28 smithi174 bash[62225]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-03-26T21:07:28.796 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:07:28 smithi174 bash[62225]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-03-26T21:07:28.796 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:07:28 smithi174 bash[62225]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-03-26T21:07:28.796 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:07:28 smithi174 bash[62225]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-03-26T21:07:28.796 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:07:28 smithi174 bash[62225]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-03-26T21:07:28.796 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:07:28 smithi174 bash[62225]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-03-26T21:07:28.796 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:07:28 smithi174 bash[62225]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-03-26T21:07:28.796 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:07:28 smithi174 bash[62225]: --> ceph-volume raw activate successful for osd ID: 6 2024-03-26T21:07:28.796 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:07:28 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6-activate[62239]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-03-26T21:07:28.796 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:07:28 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6-activate[62239]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-03-26T21:07:28.797 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:07:28 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6-activate[62239]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-03-26T21:07:28.797 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:07:28 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6-activate[62239]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-03-26T21:07:28.797 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:07:28 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6-activate[62239]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-03-26T21:07:28.797 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:07:28 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6-activate[62239]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-03-26T21:07:28.797 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:07:28 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6-activate[62239]: --> ceph-volume raw activate successful for osd ID: 6 2024-03-26T21:07:28.797 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:07:28 smithi174 podman[62225]: 2024-03-26 21:07:28.484774679 +0000 UTC m=+1.018363425 container died 4fecd3a2fcba47948e692862717f85d6f347eda605a88c6b8ca2f3562fe6cbee (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6-activate, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_CLEAN=True) 2024-03-26T21:07:28.971 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:28 smithi063 ceph-mon[66934]: pgmap v261: 97 pgs: 2 active+undersized, 11 active+undersized+degraded, 15 peering, 2 stale+active+clean, 67 active+clean; 342 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 5.4 MiB/s wr, 554 op/s; 92/1683 objects degraded (5.466%) 2024-03-26T21:07:29.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:28 smithi174 ceph-mon[53014]: pgmap v261: 97 pgs: 2 active+undersized, 11 active+undersized+degraded, 15 peering, 2 stale+active+clean, 67 active+clean; 342 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 5.4 MiB/s wr, 554 op/s; 92/1683 objects degraded (5.466%) 2024-03-26T21:07:29.439 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:07:29 smithi174 podman[62363]: 2024-03-26 21:07:29.227459923 +0000 UTC m=+0.733115989 container remove 4fecd3a2fcba47948e692862717f85d6f347eda605a88c6b8ca2f3562fe6cbee (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6-activate, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-26T21:07:29.537 INFO:teuthology.orchestra.run.smithi063.stdout: 5 278052 9.94 MB/sec execute 563 sec latency 44.644 ms 2024-03-26T21:07:29.724 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:07:29 smithi174 podman[62424]: 2024-03-26 21:07:29.350555099 +0000 UTC m=+0.019152113 image pull a9bdc3eaeb106993bcbbac35d7fe5c7b76c052021676e129f5984aed30aee063 quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d 2024-03-26T21:07:29.725 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:07:29 smithi174 podman[62424]: 2024-03-26 21:07:29.478475296 +0000 UTC m=+0.147072280 container create 95fa1b77701bf489b9459a0b0b93c37fe1c09c89a50da6d183a6fbee6d99b625 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b) 2024-03-26T21:07:30.049 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:07:29 smithi174 podman[62424]: 2024-03-26 21:07:29.73119274 +0000 UTC m=+0.399789724 container init 95fa1b77701bf489b9459a0b0b93c37fe1c09c89a50da6d183a6fbee6d99b625 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6, ceph=True, org.label-schema.build-date=20240326, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-03-26T21:07:30.049 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:07:29 smithi174 podman[62424]: 2024-03-26 21:07:29.734643427 +0000 UTC m=+0.403240420 container start 95fa1b77701bf489b9459a0b0b93c37fe1c09c89a50da6d183a6fbee6d99b625 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, ceph=True, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e) 2024-03-26T21:07:30.049 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:07:29 smithi174 ceph-osd[62442]: -- 172.21.15.174:0/674521002 <== mon.1 v2:172.21.15.174:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x560c6b00d680 con 0x560c6afd7000 2024-03-26T21:07:30.049 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:07:29 smithi174 bash[62424]: 95fa1b77701bf489b9459a0b0b93c37fe1c09c89a50da6d183a6fbee6d99b625 2024-03-26T21:07:30.049 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:07:29 smithi174 systemd[1]: Started Ceph osd.6 for 0ed4dd4c-ebb2-11ee-95d0-87774f69a715. 2024-03-26T21:07:30.537 INFO:teuthology.orchestra.run.smithi063.stdout: 5 278411 9.93 MB/sec execute 564 sec latency 37.312 ms 2024-03-26T21:07:30.548 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:07:30 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6[62438]: 2024-03-26T21:07:30.275+0000 7f05d7cb67c0 -1 Falling back to public interface 2024-03-26T21:07:31.000 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:30 smithi063 ceph-mon[66934]: pgmap v262: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 337 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 3.1 KiB/s rd, 4.1 MiB/s wr, 321 op/s; 188/1680 objects degraded (11.190%) 2024-03-26T21:07:31.000 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:30 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:07:31.000 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:30 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:07:31.000 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:30 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:07:31.000 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:30 smithi063 ceph-mon[66934]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs peering) 2024-03-26T21:07:31.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:30 smithi174 ceph-mon[53014]: pgmap v262: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 337 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 3.1 KiB/s rd, 4.1 MiB/s wr, 321 op/s; 188/1680 objects degraded (11.190%) 2024-03-26T21:07:31.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:30 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:07:31.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:30 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:07:31.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:30 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:07:31.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:30 smithi174 ceph-mon[53014]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs peering) 2024-03-26T21:07:31.537 INFO:teuthology.orchestra.run.smithi063.stdout: 5 278704 9.92 MB/sec execute 565 sec latency 40.711 ms 2024-03-26T21:07:32.537 INFO:teuthology.orchestra.run.smithi063.stdout: 5 279180 9.93 MB/sec execute 566 sec latency 40.039 ms 2024-03-26T21:07:33.003 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:32 smithi174 ceph-mon[53014]: pgmap v263: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 332 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 3.4 KiB/s rd, 4.8 MiB/s wr, 468 op/s; 181/1587 objects degraded (11.405%) 2024-03-26T21:07:33.026 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:32 smithi063 ceph-mon[66934]: pgmap v263: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 332 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 3.4 KiB/s rd, 4.8 MiB/s wr, 468 op/s; 181/1587 objects degraded (11.405%) 2024-03-26T21:07:33.537 INFO:teuthology.orchestra.run.smithi063.stdout: 5 279658 9.95 MB/sec execute 567 sec latency 44.256 ms 2024-03-26T21:07:34.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:33 smithi063 ceph-mon[66934]: Health check update: Degraded data redundancy: 181/1542 objects degraded (11.738%), 25 pgs degraded (PG_DEGRADED) 2024-03-26T21:07:34.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:33 smithi174 ceph-mon[53014]: Health check update: Degraded data redundancy: 181/1542 objects degraded (11.738%), 25 pgs degraded (PG_DEGRADED) 2024-03-26T21:07:34.537 INFO:teuthology.orchestra.run.smithi063.stdout: 5 279878 9.94 MB/sec execute 568 sec latency 72.951 ms 2024-03-26T21:07:35.078 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:35 smithi063 ceph-mon[66934]: pgmap v264: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 334 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.9 MiB/s wr, 507 op/s; 181/1542 objects degraded (11.738%) 2024-03-26T21:07:35.078 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:35 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:07:35.078 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:35 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:07:35.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:35 smithi174 ceph-mon[53014]: pgmap v264: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 334 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.9 MiB/s wr, 507 op/s; 181/1542 objects degraded (11.738%) 2024-03-26T21:07:35.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:35 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:07:35.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:35 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:07:35.537 INFO:teuthology.orchestra.run.smithi063.stdout: 5 280181 9.94 MB/sec execute 569 sec latency 60.585 ms 2024-03-26T21:07:36.486 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:36 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:07:36.486 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:36 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:07:36.486 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:36 smithi174 ceph-mon[53014]: pgmap v265: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 331 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.6 MiB/s wr, 493 op/s; 192/1560 objects degraded (12.308%) 2024-03-26T21:07:36.537 INFO:teuthology.orchestra.run.smithi063.stdout: 5 280471 9.94 MB/sec execute 570 sec latency 44.989 ms 2024-03-26T21:07:36.668 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:36 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:07:36.668 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:36 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:07:36.668 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:36 smithi063 ceph-mon[66934]: pgmap v265: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 331 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.6 MiB/s wr, 493 op/s; 192/1560 objects degraded (12.308%) 2024-03-26T21:07:37.537 INFO:teuthology.orchestra.run.smithi063.stdout: 5 280750 9.93 MB/sec execute 571 sec latency 62.945 ms 2024-03-26T21:07:38.277 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:07:37 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6[62438]: 2024-03-26T21:07:37.929+0000 7f05d7cb67c0 -1 osd.6 0 read_superblock omap replica is missing. 2024-03-26T21:07:38.537 INFO:teuthology.orchestra.run.smithi063.stdout: 5 280967 9.93 MB/sec execute 572 sec latency 56.985 ms 2024-03-26T21:07:38.856 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:38 smithi063 ceph-mon[66934]: pgmap v266: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 332 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.3 MiB/s wr, 518 op/s; 208/1665 objects degraded (12.492%) 2024-03-26T21:07:39.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:38 smithi174 ceph-mon[53014]: pgmap v266: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 332 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.3 MiB/s wr, 518 op/s; 208/1665 objects degraded (12.492%) 2024-03-26T21:07:39.537 INFO:teuthology.orchestra.run.smithi063.stdout: 5 281226 9.92 MB/sec execute 573 sec latency 51.999 ms 2024-03-26T21:07:39.798 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:07:39 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6[62438]: 2024-03-26T21:07:39.365+0000 7f05d7cb67c0 -1 osd.6 136 log_to_monitors true 2024-03-26T21:07:39.881 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:39 smithi063 ceph-mon[66934]: from='osd.6 [v2:172.21.15.174:6816/2621341956,v1:172.21.15.174:6817/2621341956]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-03-26T21:07:39.881 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:39 smithi063 ceph-mon[66934]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-03-26T21:07:39.881 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:39 smithi063 ceph-mon[66934]: Health check update: Degraded data redundancy: 208/1665 objects degraded (12.492%), 25 pgs degraded (PG_DEGRADED) 2024-03-26T21:07:40.244 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:07:39 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6[62438]: 2024-03-26T21:07:39.852+0000 7f05c910e700 -1 osd.6 136 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-26T21:07:40.245 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:39 smithi174 ceph-mon[53014]: from='osd.6 [v2:172.21.15.174:6816/2621341956,v1:172.21.15.174:6817/2621341956]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-03-26T21:07:40.245 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:39 smithi174 ceph-mon[53014]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-03-26T21:07:40.245 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:39 smithi174 ceph-mon[53014]: Health check update: Degraded data redundancy: 208/1665 objects degraded (12.492%), 25 pgs degraded (PG_DEGRADED) 2024-03-26T21:07:40.279 INFO:teuthology.orchestra.run.smithi063.stdout:true 2024-03-26T21:07:40.537 INFO:teuthology.orchestra.run.smithi063.stdout: 5 281539 9.91 MB/sec execute 574 sec latency 67.244 ms 2024-03-26T21:07:40.886 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:40 smithi063 ceph-mon[66934]: pgmap v267: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 337 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.0 MiB/s wr, 526 op/s; 204/1641 objects degraded (12.431%) 2024-03-26T21:07:40.886 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:40 smithi063 ceph-mon[66934]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-03-26T21:07:40.886 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:40 smithi063 ceph-mon[66934]: from='osd.6 [v2:172.21.15.174:6816/2621341956,v1:172.21.15.174:6817/2621341956]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]: dispatch 2024-03-26T21:07:40.886 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:40 smithi063 ceph-mon[66934]: osdmap e139: 8 total, 7 up, 8 in 2024-03-26T21:07:40.886 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:40 smithi063 ceph-mon[66934]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]: dispatch 2024-03-26T21:07:40.887 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:40 smithi063 ceph-mon[66934]: from='client.34416 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:07:40.887 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:40 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:07:40.887 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:40 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:07:40.887 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:40 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:07:40.887 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:40 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T21:07:40.887 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:40 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:07:40.887 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:40 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:07:40.887 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:40 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:07:40.887 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:40 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:07:40.887 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:40 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:07:40.887 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:40 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:07:40.887 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:40 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-03-26T21:07:40.887 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:40 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:07:40.887 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:40 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:07:41.276 INFO:teuthology.orchestra.run.smithi063.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-26T21:07:41.276 INFO:teuthology.orchestra.run.smithi063.stdout:alertmanager.smithi063 smithi063 *:9093,9094 running (8m) 2m ago 18m 24.5M - 0.23.0 ba2b418f427c 6e160c94aac3 2024-03-26T21:07:41.277 INFO:teuthology.orchestra.run.smithi063.stdout:crash.smithi063 smithi063 running (5m) 2m ago 17m 6665k - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 cf1096d84270 2024-03-26T21:07:41.277 INFO:teuthology.orchestra.run.smithi063.stdout:crash.smithi174 smithi174 running (4m) 7s ago 17m 6665k - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 cb5371a92684 2024-03-26T21:07:41.277 INFO:teuthology.orchestra.run.smithi063.stdout:grafana.smithi063 smithi063 *:3000 error 2m ago 17m - - 2024-03-26T21:07:41.277 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foofs.smithi063.tzaznz smithi063 running (12m) 2m ago 12m 37.8M - 17.2.0 e1d6a67b021e 10a989fc7954 2024-03-26T21:07:41.277 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foofs.smithi174.gitdvk smithi174 running (12m) 7s ago 12m 14.3M - 17.2.0 e1d6a67b021e 4339a5d98d1d 2024-03-26T21:07:41.277 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.smithi063.lxhujt smithi063 *:8443,9283,8765 running (7m) 2m ago 19m 509M - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 6d9d10330a90 2024-03-26T21:07:41.278 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.smithi174.sexoin smithi174 *:8443,9283,8765 running (6m) 7s ago 17m 452M - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 d7546233672d 2024-03-26T21:07:41.278 INFO:teuthology.orchestra.run.smithi063.stdout:mon.smithi063 smithi063 running (6m) 2m ago 19m 52.3M 2048M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 c644200344f8 2024-03-26T21:07:41.278 INFO:teuthology.orchestra.run.smithi063.stdout:mon.smithi174 smithi174 running (5m) 7s ago 17m 46.5M 2048M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 1a6d442692e7 2024-03-26T21:07:41.278 INFO:teuthology.orchestra.run.smithi063.stdout:nfs.foo.0.0.smithi063.fgmpwa smithi063 *:2049 running (11m) 2m ago 11m 275M - 4.0 e1d6a67b021e 3d6c0dd9ce51 2024-03-26T21:07:41.278 INFO:teuthology.orchestra.run.smithi063.stdout:nfs.foo.1.0.smithi174.pytptt smithi174 *:2049 running (11m) 7s ago 11m 86.9M - 4.0 e1d6a67b021e f220dbb449a3 2024-03-26T21:07:41.278 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.smithi063 smithi063 *:9100 running (8m) 2m ago 17m 20.7M - 1.3.1 1dbe0e931976 f52f41fd14c0 2024-03-26T21:07:41.279 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.smithi174 smithi174 *:9100 running (8m) 7s ago 17m 18.8M - 1.3.1 1dbe0e931976 0c027f39a4c8 2024-03-26T21:07:41.279 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0 smithi063 running (4m) 2m ago 16m 339M 1936M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 2d4b3fa825cf 2024-03-26T21:07:41.279 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1 smithi063 running (3m) 2m ago 16m 282M 1936M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 d41fbe507197 2024-03-26T21:07:41.279 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2 smithi063 running (2m) 2m ago 15m 200M 1936M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 f123eeabd3fa 2024-03-26T21:07:41.279 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3 smithi063 running (2m) 2m ago 15m 15.3M 1936M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 953816eafe87 2024-03-26T21:07:41.279 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4 smithi174 running (87s) 7s ago 14m 301M 2704M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 8aa35bff6cea 2024-03-26T21:07:41.279 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5 smithi174 running (49s) 7s ago 14m 267M 2704M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 25dde01e0676 2024-03-26T21:07:41.280 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6 smithi174 running (11s) 7s ago 13m 15.0M 2704M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 95fa1b77701b 2024-03-26T21:07:41.280 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7 smithi174 running (13m) 7s ago 13m 637M 2704M 17.2.0 e1d6a67b021e c1d57fcabb3f 2024-03-26T21:07:41.280 INFO:teuthology.orchestra.run.smithi063.stdout:prometheus.smithi063 smithi063 *:9095 running (7m) 2m ago 16m 76.0M - 2.33.4 514e6a882f6e 567eef202627 2024-03-26T21:07:41.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:40 smithi174 ceph-mon[53014]: pgmap v267: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 337 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.0 MiB/s wr, 526 op/s; 204/1641 objects degraded (12.431%) 2024-03-26T21:07:41.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:40 smithi174 ceph-mon[53014]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-03-26T21:07:41.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:40 smithi174 ceph-mon[53014]: from='osd.6 [v2:172.21.15.174:6816/2621341956,v1:172.21.15.174:6817/2621341956]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]: dispatch 2024-03-26T21:07:41.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:40 smithi174 ceph-mon[53014]: osdmap e139: 8 total, 7 up, 8 in 2024-03-26T21:07:41.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:40 smithi174 ceph-mon[53014]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]: dispatch 2024-03-26T21:07:41.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:40 smithi174 ceph-mon[53014]: from='client.34416 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:07:41.300 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:40 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:07:41.300 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:40 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:07:41.300 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:40 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:07:41.300 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:40 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T21:07:41.300 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:40 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:07:41.300 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:40 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:07:41.300 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:40 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:07:41.300 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:40 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:07:41.300 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:40 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:07:41.300 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:40 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:07:41.301 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:40 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-03-26T21:07:41.301 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:40 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:07:41.301 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:40 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:07:41.537 INFO:teuthology.orchestra.run.smithi063.stdout: 5 281730 9.90 MB/sec execute 575 sec latency 648.383 ms 2024-03-26T21:07:41.689 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-03-26T21:07:41.689 INFO:teuthology.orchestra.run.smithi063.stdout: "mon": { 2024-03-26T21:07:41.689 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 2 2024-03-26T21:07:41.689 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:07:41.689 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr": { 2024-03-26T21:07:41.689 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 2 2024-03-26T21:07:41.689 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:07:41.689 INFO:teuthology.orchestra.run.smithi063.stdout: "osd": { 2024-03-26T21:07:41.690 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-03-26T21:07:41.690 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 7 2024-03-26T21:07:41.690 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:07:41.690 INFO:teuthology.orchestra.run.smithi063.stdout: "mds": { 2024-03-26T21:07:41.690 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-26T21:07:41.690 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:07:41.690 INFO:teuthology.orchestra.run.smithi063.stdout: "overall": { 2024-03-26T21:07:41.690 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-03-26T21:07:41.690 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 11 2024-03-26T21:07:41.691 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-03-26T21:07:41.691 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-03-26T21:07:41.939 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:41 smithi063 ceph-mon[66934]: pgmap v269: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 337 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.1 MiB/s wr, 529 op/s; 204/1641 objects degraded (12.431%) 2024-03-26T21:07:41.939 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:41 smithi063 ceph-mon[66934]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-03-26T21:07:41.939 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:41 smithi063 ceph-mon[66934]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-03-26T21:07:41.939 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:41 smithi063 ceph-mon[66934]: from='client.34420 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:07:41.939 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:41 smithi063 ceph-mon[66934]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-26T21:07:41.939 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:41 smithi063 ceph-mon[66934]: osd.6 [v2:172.21.15.174:6816/2621341956,v1:172.21.15.174:6817/2621341956] boot 2024-03-26T21:07:41.939 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:41 smithi063 ceph-mon[66934]: osdmap e140: 8 total, 8 up, 8 in 2024-03-26T21:07:41.939 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:41 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-26T21:07:41.939 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:41 smithi063 ceph-mon[66934]: from='client.44333 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:07:41.940 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:41 smithi063 ceph-mon[66934]: from='client.? 172.21.15.63:0/1786069708' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:07:41.940 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:41 smithi063 ceph-mon[66934]: osdmap e141: 8 total, 8 up, 8 in 2024-03-26T21:07:42.129 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-03-26T21:07:42.129 INFO:teuthology.orchestra.run.smithi063.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d", 2024-03-26T21:07:42.129 INFO:teuthology.orchestra.run.smithi063.stdout: "in_progress": true, 2024-03-26T21:07:42.129 INFO:teuthology.orchestra.run.smithi063.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-26T21:07:42.130 INFO:teuthology.orchestra.run.smithi063.stdout: "services_complete": [ 2024-03-26T21:07:42.130 INFO:teuthology.orchestra.run.smithi063.stdout: "crash", 2024-03-26T21:07:42.130 INFO:teuthology.orchestra.run.smithi063.stdout: "mon", 2024-03-26T21:07:42.130 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr" 2024-03-26T21:07:42.130 INFO:teuthology.orchestra.run.smithi063.stdout: ], 2024-03-26T21:07:42.130 INFO:teuthology.orchestra.run.smithi063.stdout: "progress": "13/23 daemons upgraded", 2024-03-26T21:07:42.130 INFO:teuthology.orchestra.run.smithi063.stdout: "message": "Currently upgrading osd daemons", 2024-03-26T21:07:42.130 INFO:teuthology.orchestra.run.smithi063.stdout: "is_paused": false 2024-03-26T21:07:42.130 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-03-26T21:07:42.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:41 smithi174 ceph-mon[53014]: pgmap v269: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 337 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.1 MiB/s wr, 529 op/s; 204/1641 objects degraded (12.431%) 2024-03-26T21:07:42.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:41 smithi174 ceph-mon[53014]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-03-26T21:07:42.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:41 smithi174 ceph-mon[53014]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-03-26T21:07:42.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:41 smithi174 ceph-mon[53014]: from='client.34420 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:07:42.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:41 smithi174 ceph-mon[53014]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-26T21:07:42.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:41 smithi174 ceph-mon[53014]: osd.6 [v2:172.21.15.174:6816/2621341956,v1:172.21.15.174:6817/2621341956] boot 2024-03-26T21:07:42.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:41 smithi174 ceph-mon[53014]: osdmap e140: 8 total, 8 up, 8 in 2024-03-26T21:07:42.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:41 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-26T21:07:42.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:41 smithi174 ceph-mon[53014]: from='client.44333 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:07:42.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:41 smithi174 ceph-mon[53014]: from='client.? 172.21.15.63:0/1786069708' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:07:42.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:41 smithi174 ceph-mon[53014]: osdmap e141: 8 total, 8 up, 8 in 2024-03-26T21:07:42.537 INFO:teuthology.orchestra.run.smithi063.stdout: 5 281825 9.88 MB/sec execute 576 sec latency 1603.461 ms 2024-03-26T21:07:42.615 INFO:teuthology.orchestra.run.smithi063.stdout:HEALTH_WARN 1 failed cephadm daemon(s); Degraded data redundancy: 204/1641 objects degraded (12.431%), 25 pgs degraded 2024-03-26T21:07:42.616 INFO:teuthology.orchestra.run.smithi063.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-26T21:07:42.616 INFO:teuthology.orchestra.run.smithi063.stdout: daemon grafana.smithi063 on smithi063 is in error state 2024-03-26T21:07:42.616 INFO:teuthology.orchestra.run.smithi063.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 204/1641 objects degraded (12.431%), 25 pgs degraded 2024-03-26T21:07:42.616 INFO:teuthology.orchestra.run.smithi063.stdout: pg 1.0 is active+undersized+degraded, acting [7,0] 2024-03-26T21:07:42.616 INFO:teuthology.orchestra.run.smithi063.stdout: pg 2.2 is active+undersized+degraded, acting [5,1] 2024-03-26T21:07:42.616 INFO:teuthology.orchestra.run.smithi063.stdout: pg 2.6 is active+undersized+degraded, acting [1,4] 2024-03-26T21:07:42.616 INFO:teuthology.orchestra.run.smithi063.stdout: pg 2.7 is active+undersized+degraded, acting [7,2] 2024-03-26T21:07:42.616 INFO:teuthology.orchestra.run.smithi063.stdout: pg 2.11 is active+undersized+degraded, acting [4,1] 2024-03-26T21:07:42.616 INFO:teuthology.orchestra.run.smithi063.stdout: pg 2.14 is active+undersized+degraded, acting [3,5] 2024-03-26T21:07:42.616 INFO:teuthology.orchestra.run.smithi063.stdout: pg 2.16 is active+undersized+degraded, acting [5,2] 2024-03-26T21:07:42.616 INFO:teuthology.orchestra.run.smithi063.stdout: pg 2.17 is active+undersized+degraded, acting [5,2] 2024-03-26T21:07:42.616 INFO:teuthology.orchestra.run.smithi063.stdout: pg 2.19 is active+undersized+degraded, acting [3,0] 2024-03-26T21:07:42.616 INFO:teuthology.orchestra.run.smithi063.stdout: pg 2.1a is active+undersized+degraded, acting [4,7] 2024-03-26T21:07:42.617 INFO:teuthology.orchestra.run.smithi063.stdout: pg 2.1b is active+undersized+degraded, acting [3,7] 2024-03-26T21:07:42.617 INFO:teuthology.orchestra.run.smithi063.stdout: pg 2.1d is active+undersized+degraded, acting [7,0] 2024-03-26T21:07:42.617 INFO:teuthology.orchestra.run.smithi063.stdout: pg 3.0 is active+undersized+degraded, acting [1,2] 2024-03-26T21:07:42.617 INFO:teuthology.orchestra.run.smithi063.stdout: pg 3.2 is active+undersized+degraded, acting [3,5] 2024-03-26T21:07:42.617 INFO:teuthology.orchestra.run.smithi063.stdout: pg 3.3 is active+undersized+degraded, acting [4,0] 2024-03-26T21:07:42.617 INFO:teuthology.orchestra.run.smithi063.stdout: pg 3.a is active+undersized+degraded, acting [4,1] 2024-03-26T21:07:42.617 INFO:teuthology.orchestra.run.smithi063.stdout: pg 3.c is active+undersized+degraded, acting [5,3] 2024-03-26T21:07:42.617 INFO:teuthology.orchestra.run.smithi063.stdout: pg 3.d is active+undersized+degraded, acting [7,5] 2024-03-26T21:07:42.617 INFO:teuthology.orchestra.run.smithi063.stdout: pg 3.10 is active+undersized+degraded, acting [0,5] 2024-03-26T21:07:42.617 INFO:teuthology.orchestra.run.smithi063.stdout: pg 3.11 is active+undersized+degraded, acting [7,4] 2024-03-26T21:07:42.617 INFO:teuthology.orchestra.run.smithi063.stdout: pg 3.14 is active+undersized+degraded, acting [4,7] 2024-03-26T21:07:42.617 INFO:teuthology.orchestra.run.smithi063.stdout: pg 3.1d is active+undersized+degraded, acting [5,4] 2024-03-26T21:07:42.617 INFO:teuthology.orchestra.run.smithi063.stdout: pg 3.1e is active+undersized+degraded, acting [3,2] 2024-03-26T21:07:42.617 INFO:teuthology.orchestra.run.smithi063.stdout: pg 4.5 is active+undersized+degraded, acting [4,7] 2024-03-26T21:07:42.617 INFO:teuthology.orchestra.run.smithi063.stdout: pg 4.a is active+undersized+degraded, acting [1,7] 2024-03-26T21:07:43.114 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:42 smithi063 ceph-mon[66934]: from='client.34432 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:07:43.114 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:42 smithi063 ceph-mon[66934]: from='client.? 172.21.15.63:0/1404670466' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-26T21:07:43.114 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:42 smithi063 ceph-mon[66934]: osdmap e142: 8 total, 8 up, 8 in 2024-03-26T21:07:43.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:42 smithi174 ceph-mon[53014]: from='client.34432 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:07:43.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:42 smithi174 ceph-mon[53014]: from='client.? 172.21.15.63:0/1404670466' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-26T21:07:43.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:42 smithi174 ceph-mon[53014]: osdmap e142: 8 total, 8 up, 8 in 2024-03-26T21:07:43.537 INFO:teuthology.orchestra.run.smithi063.stdout: 5 282007 9.87 MB/sec execute 577 sec latency 1942.023 ms 2024-03-26T21:07:43.887 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:43 smithi063 ceph-mon[66934]: pgmap v272: 97 pgs: 1 remapped+peering, 10 peering, 8 active+undersized, 16 active+undersized+degraded, 62 active+clean; 329 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.3 MiB/s wr, 489 op/s; 111/1584 objects degraded (7.008%) 2024-03-26T21:07:44.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:43 smithi174 ceph-mon[53014]: pgmap v272: 97 pgs: 1 remapped+peering, 10 peering, 8 active+undersized, 16 active+undersized+degraded, 62 active+clean; 329 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.3 MiB/s wr, 489 op/s; 111/1584 objects degraded (7.008%) 2024-03-26T21:07:44.537 INFO:teuthology.orchestra.run.smithi063.stdout: 5 282284 9.87 MB/sec execute 578 sec latency 60.851 ms 2024-03-26T21:07:44.958 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:44 smithi063 ceph-mon[66934]: osdmap e143: 8 total, 8 up, 8 in 2024-03-26T21:07:44.959 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:44 smithi063 ceph-mon[66934]: Health check update: Degraded data redundancy: 111/1584 objects degraded (7.008%), 16 pgs degraded (PG_DEGRADED) 2024-03-26T21:07:44.959 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:44 smithi063 ceph-mon[66934]: osdmap e144: 8 total, 8 up, 8 in 2024-03-26T21:07:45.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:44 smithi174 ceph-mon[53014]: osdmap e143: 8 total, 8 up, 8 in 2024-03-26T21:07:45.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:44 smithi174 ceph-mon[53014]: Health check update: Degraded data redundancy: 111/1584 objects degraded (7.008%), 16 pgs degraded (PG_DEGRADED) 2024-03-26T21:07:45.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:44 smithi174 ceph-mon[53014]: osdmap e144: 8 total, 8 up, 8 in 2024-03-26T21:07:45.537 INFO:teuthology.orchestra.run.smithi063.stdout: 5 282459 9.86 MB/sec execute 579 sec latency 910.726 ms 2024-03-26T21:07:46.064 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:45 smithi063 ceph-mon[66934]: pgmap v275: 97 pgs: 1 active+recovering+degraded, 1 remapped+peering, 10 peering, 2 active+undersized, 10 active+undersized+degraded, 73 active+clean; 326 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 1.7 MiB/s wr, 129 op/s; 58/1554 objects degraded (3.732%); 321 KiB/s, 14 objects/s recovering 2024-03-26T21:07:46.065 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:45 smithi063 ceph-mon[66934]: osdmap e145: 8 total, 8 up, 8 in 2024-03-26T21:07:46.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:45 smithi174 ceph-mon[53014]: pgmap v275: 97 pgs: 1 active+recovering+degraded, 1 remapped+peering, 10 peering, 2 active+undersized, 10 active+undersized+degraded, 73 active+clean; 326 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 1.7 MiB/s wr, 129 op/s; 58/1554 objects degraded (3.732%); 321 KiB/s, 14 objects/s recovering 2024-03-26T21:07:46.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:45 smithi174 ceph-mon[53014]: osdmap e145: 8 total, 8 up, 8 in 2024-03-26T21:07:46.537 INFO:teuthology.orchestra.run.smithi063.stdout: 5 282789 9.87 MB/sec execute 580 sec latency 1037.480 ms 2024-03-26T21:07:47.537 INFO:teuthology.orchestra.run.smithi063.stdout: 5 283222 9.87 MB/sec execute 581 sec latency 413.220 ms 2024-03-26T21:07:47.833 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:47 smithi063 ceph-mon[66934]: pgmap v278: 97 pgs: 1 active+recovering+degraded, 7 peering, 89 active+clean; 330 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 2.4 MiB/s wr, 371 op/s; 4/1551 objects degraded (0.258%); 1.4 MiB/s, 53 keys/s, 36 objects/s recovering 2024-03-26T21:07:47.833 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:47 smithi063 ceph-mon[66934]: osdmap e146: 8 total, 8 up, 8 in 2024-03-26T21:07:48.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:47 smithi174 ceph-mon[53014]: pgmap v278: 97 pgs: 1 active+recovering+degraded, 7 peering, 89 active+clean; 330 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 2.4 MiB/s wr, 371 op/s; 4/1551 objects degraded (0.258%); 1.4 MiB/s, 53 keys/s, 36 objects/s recovering 2024-03-26T21:07:48.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:47 smithi174 ceph-mon[53014]: osdmap e146: 8 total, 8 up, 8 in 2024-03-26T21:07:48.537 INFO:teuthology.orchestra.run.smithi063.stdout: 5 283487 9.87 MB/sec execute 582 sec latency 45.888 ms 2024-03-26T21:07:48.857 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:48 smithi063 ceph-mon[66934]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/1551 objects degraded (0.258%), 1 pg degraded) 2024-03-26T21:07:49.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:48 smithi174 ceph-mon[53014]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/1551 objects degraded (0.258%), 1 pg degraded) 2024-03-26T21:07:49.537 INFO:teuthology.orchestra.run.smithi063.stdout: 5 283787 9.87 MB/sec execute 583 sec latency 38.483 ms 2024-03-26T21:07:49.879 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:49 smithi063 ceph-mon[66934]: pgmap v280: 97 pgs: 3 peering, 94 active+clean; 327 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.4 KiB/s rd, 3.8 MiB/s wr, 408 op/s; 2.0 MiB/s, 48 keys/s, 40 objects/s recovering 2024-03-26T21:07:50.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:49 smithi174 ceph-mon[53014]: pgmap v280: 97 pgs: 3 peering, 94 active+clean; 327 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.4 KiB/s rd, 3.8 MiB/s wr, 408 op/s; 2.0 MiB/s, 48 keys/s, 40 objects/s recovering 2024-03-26T21:07:50.538 INFO:teuthology.orchestra.run.smithi063.stdout: 5 284080 9.87 MB/sec execute 584 sec latency 49.567 ms 2024-03-26T21:07:51.537 INFO:teuthology.orchestra.run.smithi063.stdout: 5 284315 9.86 MB/sec execute 585 sec latency 60.048 ms 2024-03-26T21:07:51.926 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:51 smithi063 ceph-mon[66934]: pgmap v281: 97 pgs: 1 peering, 96 active+clean; 335 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 4.5 MiB/s wr, 538 op/s; 1.4 MiB/s, 35 keys/s, 27 objects/s recovering 2024-03-26T21:07:52.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:51 smithi174 ceph-mon[53014]: pgmap v281: 97 pgs: 1 peering, 96 active+clean; 335 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 4.5 MiB/s wr, 538 op/s; 1.4 MiB/s, 35 keys/s, 27 objects/s recovering 2024-03-26T21:07:52.537 INFO:teuthology.orchestra.run.smithi063.stdout: 5 284516 9.85 MB/sec execute 586 sec latency 54.188 ms 2024-03-26T21:07:53.537 INFO:teuthology.orchestra.run.smithi063.stdout: 5 284759 9.84 MB/sec execute 587 sec latency 61.674 ms 2024-03-26T21:07:53.699 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:53 smithi063 ceph-mon[66934]: pgmap v282: 97 pgs: 97 active+clean; 334 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.1 MiB/s wr, 474 op/s; 1.1 MiB/s, 27 keys/s, 21 objects/s recovering 2024-03-26T21:07:54.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:53 smithi174 ceph-mon[53014]: pgmap v282: 97 pgs: 97 active+clean; 334 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.1 MiB/s wr, 474 op/s; 1.1 MiB/s, 27 keys/s, 21 objects/s recovering 2024-03-26T21:07:54.537 INFO:teuthology.orchestra.run.smithi063.stdout: 5 285108 9.84 MB/sec execute 588 sec latency 56.354 ms 2024-03-26T21:07:55.537 INFO:teuthology.orchestra.run.smithi063.stdout: 5 285461 9.83 MB/sec execute 589 sec latency 59.733 ms 2024-03-26T21:07:55.795 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:55 smithi063 ceph-mon[66934]: pgmap v283: 97 pgs: 97 active+clean; 333 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.6 KiB/s rd, 4.1 MiB/s wr, 296 op/s; 960 KiB/s, 13 objects/s recovering 2024-03-26T21:07:55.795 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:55 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-03-26T21:07:55.795 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:55 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:07:55.795 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:55 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:07:56.051 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:55 smithi174 ceph-mon[53014]: pgmap v283: 97 pgs: 97 active+clean; 333 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.6 KiB/s rd, 4.1 MiB/s wr, 296 op/s; 960 KiB/s, 13 objects/s recovering 2024-03-26T21:07:56.051 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:55 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-03-26T21:07:56.051 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:55 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:07:56.051 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:55 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:07:56.538 INFO:teuthology.orchestra.run.smithi063.stdout: 5 285695 9.82 MB/sec execute 590 sec latency 50.810 ms 2024-03-26T21:07:56.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:56 smithi174 ceph-mon[53014]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-03-26T21:07:56.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:56 smithi174 ceph-mon[53014]: Upgrade: osd.7 is safe to restart 2024-03-26T21:07:56.840 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:56 smithi063 ceph-mon[66934]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-03-26T21:07:56.841 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:56 smithi063 ceph-mon[66934]: Upgrade: osd.7 is safe to restart 2024-03-26T21:07:57.538 INFO:teuthology.orchestra.run.smithi063.stdout: 5 286066 9.83 MB/sec execute 591 sec latency 44.404 ms 2024-03-26T21:07:57.867 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:57 smithi063 ceph-mon[66934]: pgmap v284: 97 pgs: 97 active+clean; 337 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.6 MiB/s wr, 456 op/s; 425 KiB/s, 7 objects/s recovering 2024-03-26T21:07:58.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:57 smithi174 ceph-mon[53014]: pgmap v284: 97 pgs: 97 active+clean; 337 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.6 MiB/s wr, 456 op/s; 425 KiB/s, 7 objects/s recovering 2024-03-26T21:07:58.538 INFO:teuthology.orchestra.run.smithi063.stdout: 5 286495 9.83 MB/sec execute 592 sec latency 48.795 ms 2024-03-26T21:07:59.391 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:59 smithi063 ceph-mon[66934]: Upgrade: Updating osd.7 2024-03-26T21:07:59.392 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:59 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:07:59.392 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:59 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-03-26T21:07:59.392 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:59 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:07:59.392 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:07:59 smithi063 ceph-mon[66934]: Deploying daemon osd.7 on smithi174 2024-03-26T21:07:59.538 INFO:teuthology.orchestra.run.smithi063.stdout: 5 286846 9.84 MB/sec execute 593 sec latency 43.549 ms 2024-03-26T21:07:59.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:59 smithi174 ceph-mon[53014]: Upgrade: Updating osd.7 2024-03-26T21:07:59.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:59 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:07:59.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:59 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-03-26T21:07:59.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:59 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:07:59.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:07:59 smithi174 ceph-mon[53014]: Deploying daemon osd.7 on smithi174 2024-03-26T21:08:00.477 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:00 smithi063 ceph-mon[66934]: pgmap v285: 97 pgs: 97 active+clean; 330 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.3 KiB/s rd, 4.6 MiB/s wr, 429 op/s; 363 KiB/s, 6 objects/s recovering 2024-03-26T21:08:00.538 INFO:teuthology.orchestra.run.smithi063.stdout: 5 287101 9.84 MB/sec execute 594 sec latency 93.207 ms 2024-03-26T21:08:00.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:00 smithi174 ceph-mon[53014]: pgmap v285: 97 pgs: 97 active+clean; 330 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.3 KiB/s rd, 4.6 MiB/s wr, 429 op/s; 363 KiB/s, 6 objects/s recovering 2024-03-26T21:08:01.538 INFO:teuthology.orchestra.run.smithi063.stdout: 5 287356 9.83 MB/sec execute 595 sec latency 52.279 ms 2024-03-26T21:08:02.015 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:01 smithi063 ceph-mon[66934]: pgmap v286: 97 pgs: 97 active+clean; 331 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.3 MiB/s wr, 469 op/s; 1.7 KiB/s, 1 objects/s recovering 2024-03-26T21:08:02.016 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:01 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:08:02.192 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:08:01 smithi174 systemd[1]: Stopping Ceph osd.7 for 0ed4dd4c-ebb2-11ee-95d0-87774f69a715... 2024-03-26T21:08:02.192 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:01 smithi174 ceph-mon[53014]: pgmap v286: 97 pgs: 97 active+clean; 331 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.3 MiB/s wr, 469 op/s; 1.7 KiB/s, 1 objects/s recovering 2024-03-26T21:08:02.192 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:01 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:08:02.538 INFO:teuthology.orchestra.run.smithi063.stdout: 5 287704 9.83 MB/sec execute 596 sec latency 39.526 ms 2024-03-26T21:08:02.548 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:08:02 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[40194]: 2024-03-26T21:08:02.249+0000 7f54b9b27700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-26T21:08:02.548 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:08:02 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[40194]: 2024-03-26T21:08:02.249+0000 7f54b9b27700 -1 osd.7 146 *** Got signal Terminated *** 2024-03-26T21:08:02.548 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:08:02 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[40194]: 2024-03-26T21:08:02.249+0000 7f54b9b27700 -1 osd.7 146 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-26T21:08:03.053 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:02 smithi063 ceph-mon[66934]: osd.7 marked itself down and dead 2024-03-26T21:08:03.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:03 smithi174 ceph-mon[53014]: osd.7 marked itself down and dead 2024-03-26T21:08:03.538 INFO:teuthology.orchestra.run.smithi063.stdout: 5 287872 9.82 MB/sec execute 597 sec latency 533.258 ms 2024-03-26T21:08:03.798 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:08:03 smithi174 podman[64241]: 2024-03-26 21:08:03.488373129 +0000 UTC m=+1.372395499 container died c1d57fcabb3f4c77f184d189ef08bbb992aaffe7976f6cafa23741551c690bce (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7, vendor=Red Hat, Inc., release=754, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, vcs-type=git, ceph=True, GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, GIT_CLEAN=True, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-03-26T21:08:04.244 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:03 smithi063 ceph-mon[66934]: pgmap v287: 97 pgs: 97 active+clean; 338 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.7 MiB/s wr, 411 op/s 2024-03-26T21:08:04.244 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:03 smithi063 ceph-mon[66934]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-26T21:08:04.244 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:03 smithi063 ceph-mon[66934]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-03-26T21:08:04.244 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:03 smithi063 ceph-mon[66934]: osdmap e147: 8 total, 7 up, 8 in 2024-03-26T21:08:04.285 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:04 smithi174 ceph-mon[53014]: pgmap v287: 97 pgs: 97 active+clean; 338 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.7 MiB/s wr, 411 op/s 2024-03-26T21:08:04.285 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:04 smithi174 ceph-mon[53014]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-26T21:08:04.285 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:04 smithi174 ceph-mon[53014]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-03-26T21:08:04.285 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:04 smithi174 ceph-mon[53014]: osdmap e147: 8 total, 7 up, 8 in 2024-03-26T21:08:04.538 INFO:teuthology.orchestra.run.smithi063.stdout: 5 287988 9.81 MB/sec execute 598 sec latency 1041.244 ms 2024-03-26T21:08:04.544 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:08:04 smithi174 podman[64241]: 2024-03-26 21:08:04.291577116 +0000 UTC m=+2.175599484 container cleanup c1d57fcabb3f4c77f184d189ef08bbb992aaffe7976f6cafa23741551c690bce (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7, io.openshift.expose-services=, GIT_CLEAN=True, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, distribution-scope=public, vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-03-26T21:08:04.545 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:08:04 smithi174 bash[64241]: ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7 2024-03-26T21:08:04.798 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:08:04 smithi174 podman[64270]: 2024-03-26 21:08:04.551529955 +0000 UTC m=+1.056081498 container remove c1d57fcabb3f4c77f184d189ef08bbb992aaffe7976f6cafa23741551c690bce (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7, io.buildah.version=1.19.8, GIT_CLEAN=True, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, maintainer=Guillaume Abrioux , RELEASE=HEAD, vcs-type=git, vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, version=8, name=centos-stream, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64) 2024-03-26T21:08:05.198 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:05 smithi174 ceph-mon[53014]: osdmap e148: 8 total, 7 up, 8 in 2024-03-26T21:08:05.199 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:08:04 smithi174 podman[64359]: 2024-03-26 21:08:04.834142607 +0000 UTC m=+0.020816105 image pull a9bdc3eaeb106993bcbbac35d7fe5c7b76c052021676e129f5984aed30aee063 quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d 2024-03-26T21:08:05.199 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:08:04 smithi174 podman[64359]: 2024-03-26 21:08:04.960813638 +0000 UTC m=+0.147487126 container create 55a0641e4e2b11a960324a890aa77ec7ca65604993ae5e91496bb948f5f4b03e (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7-deactivate, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240326, ceph=True, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b) 2024-03-26T21:08:05.256 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:05 smithi063 ceph-mon[66934]: osdmap e148: 8 total, 7 up, 8 in 2024-03-26T21:08:05.536 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:08:05 smithi174 podman[64359]: 2024-03-26 21:08:05.205610328 +0000 UTC m=+0.392283811 container init 55a0641e4e2b11a960324a890aa77ec7ca65604993ae5e91496bb948f5f4b03e (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7-deactivate, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e) 2024-03-26T21:08:05.536 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:08:05 smithi174 podman[64359]: 2024-03-26 21:08:05.209337836 +0000 UTC m=+0.396011321 container start 55a0641e4e2b11a960324a890aa77ec7ca65604993ae5e91496bb948f5f4b03e (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7-deactivate, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, org.label-schema.build-date=20240326, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-26T21:08:05.536 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:08:05 smithi174 podman[64359]: 2024-03-26 21:08:05.305585915 +0000 UTC m=+0.492259425 container attach 55a0641e4e2b11a960324a890aa77ec7ca65604993ae5e91496bb948f5f4b03e (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7-deactivate, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240326, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, ceph=True) 2024-03-26T21:08:05.538 INFO:teuthology.orchestra.run.smithi063.stdout: 5 288258 9.81 MB/sec execute 599 sec latency 47.812 ms 2024-03-26T21:08:05.798 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:08:05 smithi174 podman[64359]: 2024-03-26 21:08:05.543143331 +0000 UTC m=+0.729816816 container died 55a0641e4e2b11a960324a890aa77ec7ca65604993ae5e91496bb948f5f4b03e (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7-deactivate, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, org.label-schema.vendor=CentOS, ceph=True) 2024-03-26T21:08:06.027 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:06 smithi063 ceph-mon[66934]: pgmap v290: 97 pgs: 8 active+undersized+degraded, 8 stale+active+clean, 81 active+clean; 341 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 4.0 KiB/s rd, 4.3 MiB/s wr, 311 op/s; 93/1692 objects degraded (5.496%) 2024-03-26T21:08:06.027 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:06 smithi063 ceph-mon[66934]: Health check failed: Degraded data redundancy: 93/1692 objects degraded (5.496%), 8 pgs degraded (PG_DEGRADED) 2024-03-26T21:08:06.178 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:06 smithi174 ceph-mon[53014]: pgmap v290: 97 pgs: 8 active+undersized+degraded, 8 stale+active+clean, 81 active+clean; 341 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 4.0 KiB/s rd, 4.3 MiB/s wr, 311 op/s; 93/1692 objects degraded (5.496%) 2024-03-26T21:08:06.178 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:06 smithi174 ceph-mon[53014]: Health check failed: Degraded data redundancy: 93/1692 objects degraded (5.496%), 8 pgs degraded (PG_DEGRADED) 2024-03-26T21:08:06.432 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:08:06 smithi174 podman[64384]: 2024-03-26 21:08:06.184922079 +0000 UTC m=+0.635546903 container remove 55a0641e4e2b11a960324a890aa77ec7ca65604993ae5e91496bb948f5f4b03e (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7-deactivate, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-26T21:08:06.432 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:08:06 smithi174 systemd[1]: ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715@osd.7.service: Deactivated successfully. 2024-03-26T21:08:06.432 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:08:06 smithi174 systemd[1]: Stopped Ceph osd.7 for 0ed4dd4c-ebb2-11ee-95d0-87774f69a715. 2024-03-26T21:08:06.433 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:08:06 smithi174 systemd[1]: ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715@osd.7.service: Consumed 7min 42.019s CPU time. 2024-03-26T21:08:06.538 INFO:teuthology.orchestra.run.smithi063.stdout: 5 cleanup 600 sec 2024-03-26T21:08:06.687 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:08:06 smithi174 systemd[1]: Starting Ceph osd.7 for 0ed4dd4c-ebb2-11ee-95d0-87774f69a715... 2024-03-26T21:08:06.687 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:08:06 smithi174 podman[64482]: 2024-03-26 21:08:06.570563604 +0000 UTC m=+0.019789606 image pull a9bdc3eaeb106993bcbbac35d7fe5c7b76c052021676e129f5984aed30aee063 quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d 2024-03-26T21:08:06.687 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:08:06 smithi174 podman[64482]: 2024-03-26 21:08:06.69427165 +0000 UTC m=+0.143497649 container create b49058907b5abc31cf38f9ada6033426abb1c929c213ee712d1d9fed53b2e5fc (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7-activate, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326, io.buildah.version=1.33.5) 2024-03-26T21:08:06.732 INFO:teuthology.orchestra.run.smithi063.stdout:[289376] unlink './clients/client1/~dmtmp/PARADOX/.nfs00000100000002d700000007' failed - Device or resource busy 2024-03-26T21:08:06.732 INFO:teuthology.orchestra.run.smithi063.stdout:[289376] unlink './clients/client1/~dmtmp/PARADOX/.nfs00000100000002ff00000004' failed - Device or resource busy 2024-03-26T21:08:06.732 INFO:teuthology.orchestra.run.smithi063.stdout:[289376] unlink './clients/client1/~dmtmp/PARADOX/.nfs00000100000002f000000011' failed - Device or resource busy 2024-03-26T21:08:06.733 INFO:teuthology.orchestra.run.smithi063.stdout:[289376] unlink './clients/client1/~dmtmp/PARADOX/.nfs00000100000002bb00000014' failed - Device or resource busy 2024-03-26T21:08:06.734 INFO:teuthology.orchestra.run.smithi063.stdout:[289376] unlink './clients/client1/~dmtmp/PARADOX/.nfs00000100000002c000000001' failed - Device or resource busy 2024-03-26T21:08:06.734 INFO:teuthology.orchestra.run.smithi063.stdout:[289376] unlink './clients/client1/~dmtmp/PARADOX/.nfs000001000000030300000018' failed - Device or resource busy 2024-03-26T21:08:06.735 INFO:teuthology.orchestra.run.smithi063.stdout:[289376] unlink './clients/client1/~dmtmp/PARADOX/.nfs00000100000002f40000001a' failed - Device or resource busy 2024-03-26T21:08:06.766 INFO:teuthology.orchestra.run.smithi063.stdout:[289376] unlink './clients/client1/~dmtmp/PARADOX/.nfs00000100000002e90000000d' failed - Device or resource busy 2024-03-26T21:08:06.766 INFO:teuthology.orchestra.run.smithi063.stdout:[289376] unlink './clients/client1/~dmtmp/PARADOX/.nfs00000100000002c600000002' failed - Device or resource busy 2024-03-26T21:08:06.768 INFO:teuthology.orchestra.run.smithi063.stdout:[289376] unlink './clients/client1/~dmtmp/PARADOX/.nfs000001000000031200000009' failed - Device or resource busy 2024-03-26T21:08:06.770 INFO:teuthology.orchestra.run.smithi063.stdout:[289376] unlink './clients/client1/~dmtmp/PARADOX/.nfs00000100000003080000000e' failed - Device or resource busy 2024-03-26T21:08:06.770 INFO:teuthology.orchestra.run.smithi063.stdout:[289376] unlink './clients/client1/~dmtmp/PARADOX/.nfs00000100000002df00000003' failed - Device or resource busy 2024-03-26T21:08:06.773 INFO:teuthology.orchestra.run.smithi063.stdout:[289376] unlink './clients/client1/~dmtmp/PARADOX/.nfs00000100000003170000000b' failed - Device or resource busy 2024-03-26T21:08:06.776 INFO:teuthology.orchestra.run.smithi063.stdout:[289376] unlink './clients/client1/~dmtmp/PARADOX/.nfs00000100000002ce00000015' failed - Device or resource busy 2024-03-26T21:08:06.836 INFO:teuthology.orchestra.run.smithi063.stdout: 0 cleanup 600 sec 2024-03-26T21:08:06.836 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-03-26T21:08:06.836 INFO:teuthology.orchestra.run.smithi063.stdout: Operation Count AvgLat MaxLat 2024-03-26T21:08:06.837 INFO:teuthology.orchestra.run.smithi063.stdout: ---------------------------------------- 2024-03-26T21:08:06.837 INFO:teuthology.orchestra.run.smithi063.stdout: NTCreateX 188894 4.583 2168.927 2024-03-26T21:08:06.837 INFO:teuthology.orchestra.run.smithi063.stdout: Close 138553 0.142 11.163 2024-03-26T21:08:06.837 INFO:teuthology.orchestra.run.smithi063.stdout: Rename 7997 9.594 2421.786 2024-03-26T21:08:06.837 INFO:teuthology.orchestra.run.smithi063.stdout: Unlink 38275 6.913 1051.559 2024-03-26T21:08:06.837 INFO:teuthology.orchestra.run.smithi063.stdout: Qpathinfo 171158 1.215 2329.811 2024-03-26T21:08:06.837 INFO:teuthology.orchestra.run.smithi063.stdout: Qfileinfo 29789 0.008 1.087 2024-03-26T21:08:06.837 INFO:teuthology.orchestra.run.smithi063.stdout: Qfsinfo 31420 0.640 27.011 2024-03-26T21:08:06.837 INFO:teuthology.orchestra.run.smithi063.stdout: Sfileinfo 15377 12.626 1067.268 2024-03-26T21:08:06.837 INFO:teuthology.orchestra.run.smithi063.stdout: Find 66105 1.011 16.846 2024-03-26T21:08:06.837 INFO:teuthology.orchestra.run.smithi063.stdout: WriteX 93159 13.710 2337.795 2024-03-26T21:08:06.837 INFO:teuthology.orchestra.run.smithi063.stdout: ReadX 296003 0.010 24.436 2024-03-26T21:08:06.838 INFO:teuthology.orchestra.run.smithi063.stdout: LockX 614 0.381 2.862 2024-03-26T21:08:06.838 INFO:teuthology.orchestra.run.smithi063.stdout: UnlockX 614 0.332 2.239 2024-03-26T21:08:06.838 INFO:teuthology.orchestra.run.smithi063.stdout: Flush 13237 0.002 0.030 2024-03-26T21:08:06.838 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-03-26T21:08:06.838 INFO:teuthology.orchestra.run.smithi063.stdout:Throughput 9.80619 MB/sec 5 clients 5 procs max_latency=2421.798 ms 2024-03-26T21:08:06.839 DEBUG:teuthology.orchestra.run.smithi063:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'umount /mnt/foo' 2024-03-26T21:08:06.880 DEBUG:teuthology.orchestra.run.smithi063:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'while ! mount -t nfs $(hostname):/fake /mnt/foo ; do sleep 5 ; done' 2024-03-26T21:08:06.995 DEBUG:teuthology.orchestra.run.smithi063:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'cd /mnt/foo && dbench 5 -t 5' 2024-03-26T21:08:07.006 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:08:06 smithi174 podman[64482]: 2024-03-26 21:08:06.930504854 +0000 UTC m=+0.379730852 container init b49058907b5abc31cf38f9ada6033426abb1c929c213ee712d1d9fed53b2e5fc (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7-activate, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240326, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b) 2024-03-26T21:08:07.007 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:08:06 smithi174 podman[64482]: 2024-03-26 21:08:06.933651451 +0000 UTC m=+0.382877453 container start b49058907b5abc31cf38f9ada6033426abb1c929c213ee712d1d9fed53b2e5fc (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7-activate, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, org.label-schema.license=GPLv2, org.label-schema.build-date=20240326, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, org.label-schema.schema-version=1.0) 2024-03-26T21:08:07.007 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:08:07 smithi174 podman[64482]: 2024-03-26 21:08:07.013638347 +0000 UTC m=+0.462864347 container attach b49058907b5abc31cf38f9ada6033426abb1c929c213ee712d1d9fed53b2e5fc (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7-activate, GIT_BRANCH=HEAD, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-03-26T21:08:07.021 INFO:teuthology.orchestra.run.smithi063.stdout:dbench version 4.00 - Copyright Andrew Tridgell 1999-2004 2024-03-26T21:08:07.022 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-03-26T21:08:07.022 INFO:teuthology.orchestra.run.smithi063.stdout:Running for 5 seconds with load '/usr/share/dbench/client.txt' and minimum warmup 1 secs 2024-03-26T21:08:07.022 INFO:teuthology.orchestra.run.smithi063.stdout:3 of 5 processes prepared for launch 0 sec 2024-03-26T21:08:07.123 INFO:teuthology.orchestra.run.smithi063.stdout:5 of 5 processes prepared for launch 0 sec 2024-03-26T21:08:07.123 INFO:teuthology.orchestra.run.smithi063.stdout:releasing clients 2024-03-26T21:08:07.798 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:08:07 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7-activate[64497]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-03-26T21:08:07.799 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:08:07 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7-activate[64497]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-03-26T21:08:07.799 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:08:07 smithi174 bash[64482]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-03-26T21:08:07.799 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:08:07 smithi174 bash[64482]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-03-26T21:08:07.799 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:08:07 smithi174 bash[64482]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-03-26T21:08:07.799 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:08:07 smithi174 bash[64482]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-03-26T21:08:07.799 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:08:07 smithi174 bash[64482]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-03-26T21:08:07.799 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:08:07 smithi174 bash[64482]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-03-26T21:08:07.799 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:08:07 smithi174 bash[64482]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-03-26T21:08:07.799 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:08:07 smithi174 bash[64482]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-03-26T21:08:07.799 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:08:07 smithi174 bash[64482]: --> ceph-volume raw activate successful for osd ID: 7 2024-03-26T21:08:07.799 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:08:07 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7-activate[64497]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-03-26T21:08:07.800 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:08:07 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7-activate[64497]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-03-26T21:08:07.800 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:08:07 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7-activate[64497]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-03-26T21:08:07.800 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:08:07 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7-activate[64497]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-03-26T21:08:07.800 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:08:07 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7-activate[64497]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-03-26T21:08:07.800 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:08:07 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7-activate[64497]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-03-26T21:08:07.800 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:08:07 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7-activate[64497]: --> ceph-volume raw activate successful for osd ID: 7 2024-03-26T21:08:07.800 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:08:07 smithi174 podman[64482]: 2024-03-26 21:08:07.469351487 +0000 UTC m=+0.918577489 container died b49058907b5abc31cf38f9ada6033426abb1c929c213ee712d1d9fed53b2e5fc (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7-activate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e) 2024-03-26T21:08:08.244 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:08 smithi063 ceph-mon[66934]: pgmap v291: 97 pgs: 2 active+undersized, 24 active+undersized+degraded, 5 stale+active+clean, 66 active+clean; 337 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 4.6 MiB/s wr, 448 op/s; 176/1656 objects degraded (10.628%) 2024-03-26T21:08:08.252 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:08 smithi174 ceph-mon[53014]: pgmap v291: 97 pgs: 2 active+undersized, 24 active+undersized+degraded, 5 stale+active+clean, 66 active+clean; 337 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 4.6 MiB/s wr, 448 op/s; 176/1656 objects degraded (10.628%) 2024-03-26T21:08:08.512 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:08:08 smithi174 podman[64635]: 2024-03-26 21:08:08.259702887 +0000 UTC m=+0.780284898 container remove b49058907b5abc31cf38f9ada6033426abb1c929c213ee712d1d9fed53b2e5fc (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7-activate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, GIT_CLEAN=True, io.buildah.version=1.33.5) 2024-03-26T21:08:08.512 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:08:08 smithi174 podman[64693]: 2024-03-26 21:08:08.395835315 +0000 UTC m=+0.021461627 image pull a9bdc3eaeb106993bcbbac35d7fe5c7b76c052021676e129f5984aed30aee063 quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d 2024-03-26T21:08:08.769 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:08:08 smithi174 podman[64693]: 2024-03-26 21:08:08.519121631 +0000 UTC m=+0.144747916 container create cb89961bc93097f840158ac51af5b6a14029f0c99bb1db5ee15ff76bc93b7374 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, GIT_CLEAN=True, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e) 2024-03-26T21:08:08.769 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:08:08 smithi174 podman[64693]: 2024-03-26 21:08:08.747255809 +0000 UTC m=+0.372882119 container init cb89961bc93097f840158ac51af5b6a14029f0c99bb1db5ee15ff76bc93b7374 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, ceph=True, org.label-schema.license=GPLv2, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-03-26T21:08:08.769 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:08:08 smithi174 podman[64693]: 2024-03-26 21:08:08.75151425 +0000 UTC m=+0.377140538 container start cb89961bc93097f840158ac51af5b6a14029f0c99bb1db5ee15ff76bc93b7374 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, ceph=True, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-03-26T21:08:09.048 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:08:08 smithi174 ceph-osd[64711]: -- 172.21.15.174:0/2318714995 <== mon.1 v2:172.21.15.174:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55e65eb51680 con 0x55e65eb1b000 2024-03-26T21:08:09.048 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:08:08 smithi174 bash[64693]: cb89961bc93097f840158ac51af5b6a14029f0c99bb1db5ee15ff76bc93b7374 2024-03-26T21:08:09.048 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:08:08 smithi174 systemd[1]: Started Ceph osd.7 for 0ed4dd4c-ebb2-11ee-95d0-87774f69a715. 2024-03-26T21:08:09.123 INFO:teuthology.orchestra.run.smithi063.stdout: 5 314 22.89 MB/sec execute 1 sec latency 141.168 ms 2024-03-26T21:08:09.548 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:08:09 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:08:09.294+0000 7feba75627c0 -1 Falling back to public interface 2024-03-26T21:08:10.098 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:09 smithi063 ceph-mon[66934]: pgmap v292: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 337 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.7 MiB/s wr, 439 op/s; 250/1566 objects degraded (15.964%) 2024-03-26T21:08:10.098 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:09 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:08:10.098 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:09 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:08:10.098 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:09 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:08:10.123 INFO:teuthology.orchestra.run.smithi063.stdout: 5 454 22.50 MB/sec execute 2 sec latency 44.224 ms 2024-03-26T21:08:10.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:09 smithi174 ceph-mon[53014]: pgmap v292: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 337 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.7 MiB/s wr, 439 op/s; 250/1566 objects degraded (15.964%) 2024-03-26T21:08:10.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:09 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:08:10.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:09 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:08:10.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:09 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:08:11.123 INFO:teuthology.orchestra.run.smithi063.stdout: 5 714 23.89 MB/sec execute 3 sec latency 36.333 ms 2024-03-26T21:08:11.882 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:11 smithi063 ceph-mon[66934]: pgmap v293: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 329 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.9 MiB/s wr, 476 op/s; 213/1425 objects degraded (14.947%) 2024-03-26T21:08:11.882 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:11 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:08:11.882 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:11 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:08:11.882 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:11 smithi063 ceph-mon[66934]: Health check update: Degraded data redundancy: 213/1425 objects degraded (14.947%), 35 pgs degraded (PG_DEGRADED) 2024-03-26T21:08:12.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:11 smithi174 ceph-mon[53014]: pgmap v293: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 329 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.9 MiB/s wr, 476 op/s; 213/1425 objects degraded (14.947%) 2024-03-26T21:08:12.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:11 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:08:12.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:11 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:08:12.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:11 smithi174 ceph-mon[53014]: Health check update: Degraded data redundancy: 213/1425 objects degraded (14.947%), 35 pgs degraded (PG_DEGRADED) 2024-03-26T21:08:12.123 INFO:teuthology.orchestra.run.smithi063.stdout: 5 992 18.95 MB/sec execute 4 sec latency 49.140 ms 2024-03-26T21:08:13.109 INFO:teuthology.orchestra.run.smithi063.stdout:true 2024-03-26T21:08:13.123 INFO:teuthology.orchestra.run.smithi063.stdout: 5 cleanup 5 sec 2024-03-26T21:08:13.410 INFO:teuthology.orchestra.run.smithi063.stdout: 0 cleanup 5 sec 2024-03-26T21:08:13.411 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-03-26T21:08:13.411 INFO:teuthology.orchestra.run.smithi063.stdout: Operation Count AvgLat MaxLat 2024-03-26T21:08:13.411 INFO:teuthology.orchestra.run.smithi063.stdout: ---------------------------------------- 2024-03-26T21:08:13.411 INFO:teuthology.orchestra.run.smithi063.stdout: NTCreateX 812 8.233 51.204 2024-03-26T21:08:13.411 INFO:teuthology.orchestra.run.smithi063.stdout: Close 747 8.259 78.223 2024-03-26T21:08:13.411 INFO:teuthology.orchestra.run.smithi063.stdout: Rename 50 13.431 49.132 2024-03-26T21:08:13.411 INFO:teuthology.orchestra.run.smithi063.stdout: Unlink 81 3.679 22.671 2024-03-26T21:08:13.411 INFO:teuthology.orchestra.run.smithi063.stdout: Qpathinfo 462 1.651 25.584 2024-03-26T21:08:13.411 INFO:teuthology.orchestra.run.smithi063.stdout: Qfileinfo 369 16.721 49.729 2024-03-26T21:08:13.411 INFO:teuthology.orchestra.run.smithi063.stdout: Qfsinfo 159 0.438 5.963 2024-03-26T21:08:13.411 INFO:teuthology.orchestra.run.smithi063.stdout: Sfileinfo 165 15.177 28.662 2024-03-26T21:08:13.411 INFO:teuthology.orchestra.run.smithi063.stdout: Find 215 0.745 2.192 2024-03-26T21:08:13.411 INFO:teuthology.orchestra.run.smithi063.stdout: WriteX 1770 0.024 0.085 2024-03-26T21:08:13.412 INFO:teuthology.orchestra.run.smithi063.stdout: ReadX 931 0.017 0.409 2024-03-26T21:08:13.412 INFO:teuthology.orchestra.run.smithi063.stdout: Flush 120 13.310 29.755 2024-03-26T21:08:13.412 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-03-26T21:08:13.412 INFO:teuthology.orchestra.run.smithi063.stdout:Throughput 18.9501 MB/sec 5 clients 5 procs max_latency=141.168 ms 2024-03-26T21:08:13.413 DEBUG:teuthology.parallel:result is None 2024-03-26T21:08:13.887 INFO:teuthology.orchestra.run.smithi063.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-26T21:08:13.887 INFO:teuthology.orchestra.run.smithi063.stdout:alertmanager.smithi063 smithi063 *:9093,9094 running (9m) 2m ago 18m 24.5M - 0.23.0 ba2b418f427c 6e160c94aac3 2024-03-26T21:08:13.887 INFO:teuthology.orchestra.run.smithi063.stdout:crash.smithi063 smithi063 running (5m) 2m ago 18m 6665k - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 cf1096d84270 2024-03-26T21:08:13.887 INFO:teuthology.orchestra.run.smithi063.stdout:crash.smithi174 smithi174 running (5m) 0s ago 17m 6665k - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 cb5371a92684 2024-03-26T21:08:13.887 INFO:teuthology.orchestra.run.smithi063.stdout:grafana.smithi063 smithi063 *:3000 error 2m ago 18m - - 2024-03-26T21:08:13.887 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foofs.smithi063.tzaznz smithi063 running (12m) 2m ago 12m 37.8M - 17.2.0 e1d6a67b021e 10a989fc7954 2024-03-26T21:08:13.887 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foofs.smithi174.gitdvk smithi174 running (12m) 0s ago 12m 14.4M - 17.2.0 e1d6a67b021e 4339a5d98d1d 2024-03-26T21:08:13.888 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.smithi063.lxhujt smithi063 *:8443,9283,8765 running (8m) 2m ago 19m 509M - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 6d9d10330a90 2024-03-26T21:08:13.888 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.smithi174.sexoin smithi174 *:8443,9283,8765 running (7m) 0s ago 17m 453M - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 d7546233672d 2024-03-26T21:08:13.888 INFO:teuthology.orchestra.run.smithi063.stdout:mon.smithi063 smithi063 running (7m) 2m ago 19m 52.3M 2048M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 c644200344f8 2024-03-26T21:08:13.888 INFO:teuthology.orchestra.run.smithi063.stdout:mon.smithi174 smithi174 running (6m) 0s ago 17m 47.9M 2048M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 1a6d442692e7 2024-03-26T21:08:13.888 INFO:teuthology.orchestra.run.smithi063.stdout:nfs.foo.0.0.smithi063.fgmpwa smithi063 *:2049 running (12m) 2m ago 12m 275M - 4.0 e1d6a67b021e 3d6c0dd9ce51 2024-03-26T21:08:13.888 INFO:teuthology.orchestra.run.smithi063.stdout:nfs.foo.1.0.smithi174.pytptt smithi174 *:2049 running (12m) 0s ago 12m 86.9M - 4.0 e1d6a67b021e f220dbb449a3 2024-03-26T21:08:13.888 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.smithi063 smithi063 *:9100 running (9m) 2m ago 18m 20.7M - 1.3.1 1dbe0e931976 f52f41fd14c0 2024-03-26T21:08:13.888 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.smithi174 smithi174 *:9100 running (8m) 0s ago 17m 18.4M - 1.3.1 1dbe0e931976 0c027f39a4c8 2024-03-26T21:08:13.888 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0 smithi063 running (4m) 2m ago 17m 339M 1936M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 2d4b3fa825cf 2024-03-26T21:08:13.888 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1 smithi063 running (4m) 2m ago 16m 282M 1936M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 d41fbe507197 2024-03-26T21:08:13.888 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2 smithi063 running (3m) 2m ago 16m 200M 1936M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 f123eeabd3fa 2024-03-26T21:08:13.888 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3 smithi063 running (2m) 2m ago 15m 15.3M 1936M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 953816eafe87 2024-03-26T21:08:13.888 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4 smithi174 running (119s) 0s ago 15m 349M 2704M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 8aa35bff6cea 2024-03-26T21:08:13.888 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5 smithi174 running (82s) 0s ago 14m 307M 2704M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 25dde01e0676 2024-03-26T21:08:13.888 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6 smithi174 running (44s) 0s ago 14m 286M 2704M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 95fa1b77701b 2024-03-26T21:08:13.889 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7 smithi174 running (5s) 0s ago 14m 12.7M 2704M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 cb89961bc930 2024-03-26T21:08:13.889 INFO:teuthology.orchestra.run.smithi063.stdout:prometheus.smithi063 smithi063 *:9095 running (7m) 2m ago 17m 76.0M - 2.33.4 514e6a882f6e 567eef202627 2024-03-26T21:08:14.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:13 smithi174 ceph-mon[53014]: pgmap v294: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 324 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 10 MiB/s wr, 445 op/s; 200/1314 objects degraded (15.221%) 2024-03-26T21:08:14.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:13 smithi174 ceph-mon[53014]: from='client.34442 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:08:14.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:13 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:08:14.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:13 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:08:14.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:13 smithi063 ceph-mon[66934]: pgmap v294: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 324 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 10 MiB/s wr, 445 op/s; 200/1314 objects degraded (15.221%) 2024-03-26T21:08:14.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:13 smithi063 ceph-mon[66934]: from='client.34442 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:08:14.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:13 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:08:14.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:13 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:08:14.311 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-03-26T21:08:14.311 INFO:teuthology.orchestra.run.smithi063.stdout: "mon": { 2024-03-26T21:08:14.312 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 2 2024-03-26T21:08:14.312 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:08:14.312 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr": { 2024-03-26T21:08:14.312 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 2 2024-03-26T21:08:14.312 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:08:14.312 INFO:teuthology.orchestra.run.smithi063.stdout: "osd": { 2024-03-26T21:08:14.312 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 7 2024-03-26T21:08:14.312 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:08:14.312 INFO:teuthology.orchestra.run.smithi063.stdout: "mds": { 2024-03-26T21:08:14.312 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-03-26T21:08:14.312 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:08:14.312 INFO:teuthology.orchestra.run.smithi063.stdout: "overall": { 2024-03-26T21:08:14.312 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-03-26T21:08:14.312 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 11 2024-03-26T21:08:14.313 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-03-26T21:08:14.313 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-03-26T21:08:14.669 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-03-26T21:08:14.669 INFO:teuthology.orchestra.run.smithi063.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d", 2024-03-26T21:08:14.670 INFO:teuthology.orchestra.run.smithi063.stdout: "in_progress": true, 2024-03-26T21:08:14.670 INFO:teuthology.orchestra.run.smithi063.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-26T21:08:14.670 INFO:teuthology.orchestra.run.smithi063.stdout: "services_complete": [ 2024-03-26T21:08:14.670 INFO:teuthology.orchestra.run.smithi063.stdout: "osd", 2024-03-26T21:08:14.670 INFO:teuthology.orchestra.run.smithi063.stdout: "crash", 2024-03-26T21:08:14.670 INFO:teuthology.orchestra.run.smithi063.stdout: "mon", 2024-03-26T21:08:14.670 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr" 2024-03-26T21:08:14.670 INFO:teuthology.orchestra.run.smithi063.stdout: ], 2024-03-26T21:08:14.670 INFO:teuthology.orchestra.run.smithi063.stdout: "progress": "14/23 daemons upgraded", 2024-03-26T21:08:14.670 INFO:teuthology.orchestra.run.smithi063.stdout: "message": "Currently upgrading osd daemons", 2024-03-26T21:08:14.670 INFO:teuthology.orchestra.run.smithi063.stdout: "is_paused": false 2024-03-26T21:08:14.670 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-03-26T21:08:15.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:14 smithi174 ceph-mon[53014]: from='client.34446 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:08:15.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:14 smithi174 ceph-mon[53014]: from='client.34450 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:08:15.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:14 smithi174 ceph-mon[53014]: from='client.? 172.21.15.63:0/985848249' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:08:15.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:14 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:08:15.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:14 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:08:15.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:14 smithi063 ceph-mon[66934]: from='client.34446 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:08:15.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:14 smithi063 ceph-mon[66934]: from='client.34450 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:08:15.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:14 smithi063 ceph-mon[66934]: from='client.? 172.21.15.63:0/985848249' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:08:15.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:14 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:08:15.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:14 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:08:15.108 INFO:teuthology.orchestra.run.smithi063.stdout:HEALTH_WARN 1 failed cephadm daemon(s); 1 osds down; all OSDs are running squid or later but require_osd_release < squid; Degraded data redundancy: 201/1323 objects degraded (15.193%), 35 pgs degraded 2024-03-26T21:08:15.109 INFO:teuthology.orchestra.run.smithi063.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-26T21:08:15.109 INFO:teuthology.orchestra.run.smithi063.stdout: daemon grafana.smithi063 on smithi063 is in error state 2024-03-26T21:08:15.109 INFO:teuthology.orchestra.run.smithi063.stdout:[WRN] OSD_DOWN: 1 osds down 2024-03-26T21:08:15.109 INFO:teuthology.orchestra.run.smithi063.stdout: osd.7 (root=default,host=smithi174) is down 2024-03-26T21:08:15.109 INFO:teuthology.orchestra.run.smithi063.stdout:[WRN] OSD_UPGRADE_FINISHED: all OSDs are running squid or later but require_osd_release < squid 2024-03-26T21:08:15.109 INFO:teuthology.orchestra.run.smithi063.stdout: all OSDs are running squid or later but require_osd_release < squid 2024-03-26T21:08:15.109 INFO:teuthology.orchestra.run.smithi063.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 201/1323 objects degraded (15.193%), 35 pgs degraded 2024-03-26T21:08:15.109 INFO:teuthology.orchestra.run.smithi063.stdout: pg 1.0 is active+undersized+degraded, acting [0,6] 2024-03-26T21:08:15.109 INFO:teuthology.orchestra.run.smithi063.stdout: pg 2.0 is active+undersized+degraded, acting [1,0] 2024-03-26T21:08:15.109 INFO:teuthology.orchestra.run.smithi063.stdout: pg 2.4 is active+undersized+degraded, acting [1,0] 2024-03-26T21:08:15.109 INFO:teuthology.orchestra.run.smithi063.stdout: pg 2.5 is active+undersized+degraded, acting [0,4] 2024-03-26T21:08:15.109 INFO:teuthology.orchestra.run.smithi063.stdout: pg 2.7 is active+undersized+degraded, acting [6,2] 2024-03-26T21:08:15.109 INFO:teuthology.orchestra.run.smithi063.stdout: pg 2.8 is active+undersized+degraded, acting [0,1] 2024-03-26T21:08:15.109 INFO:teuthology.orchestra.run.smithi063.stdout: pg 2.9 is active+undersized+degraded, acting [1,3] 2024-03-26T21:08:15.110 INFO:teuthology.orchestra.run.smithi063.stdout: pg 2.a is active+undersized+degraded, acting [1,3] 2024-03-26T21:08:15.110 INFO:teuthology.orchestra.run.smithi063.stdout: pg 2.b is active+undersized+degraded, acting [4,5] 2024-03-26T21:08:15.110 INFO:teuthology.orchestra.run.smithi063.stdout: pg 2.e is active+undersized+degraded, acting [2,3] 2024-03-26T21:08:15.110 INFO:teuthology.orchestra.run.smithi063.stdout: pg 2.f is active+undersized+degraded, acting [4,0] 2024-03-26T21:08:15.110 INFO:teuthology.orchestra.run.smithi063.stdout: pg 2.12 is active+undersized+degraded, acting [5,3] 2024-03-26T21:08:15.110 INFO:teuthology.orchestra.run.smithi063.stdout: pg 2.18 is active+undersized+degraded, acting [5,4] 2024-03-26T21:08:15.110 INFO:teuthology.orchestra.run.smithi063.stdout: pg 2.1a is active+undersized+degraded, acting [6,4] 2024-03-26T21:08:15.110 INFO:teuthology.orchestra.run.smithi063.stdout: pg 2.1b is active+undersized+degraded, acting [3,6] 2024-03-26T21:08:15.110 INFO:teuthology.orchestra.run.smithi063.stdout: pg 2.1c is active+undersized+degraded, acting [5,2] 2024-03-26T21:08:15.110 INFO:teuthology.orchestra.run.smithi063.stdout: pg 2.1d is active+undersized+degraded, acting [6,0] 2024-03-26T21:08:15.110 INFO:teuthology.orchestra.run.smithi063.stdout: pg 2.1f is active+undersized+degraded, acting [0,4] 2024-03-26T21:08:15.110 INFO:teuthology.orchestra.run.smithi063.stdout: pg 3.7 is active+undersized+degraded, acting [3,0] 2024-03-26T21:08:15.110 INFO:teuthology.orchestra.run.smithi063.stdout: pg 3.8 is active+undersized+degraded, acting [3,1] 2024-03-26T21:08:15.111 INFO:teuthology.orchestra.run.smithi063.stdout: pg 3.9 is active+undersized+degraded, acting [4,2] 2024-03-26T21:08:15.111 INFO:teuthology.orchestra.run.smithi063.stdout: pg 3.d is active+undersized+degraded, acting [5,6] 2024-03-26T21:08:15.111 INFO:teuthology.orchestra.run.smithi063.stdout: pg 3.e is active+undersized+degraded, acting [4,1] 2024-03-26T21:08:15.111 INFO:teuthology.orchestra.run.smithi063.stdout: pg 3.f is active+undersized+degraded, acting [4,0] 2024-03-26T21:08:15.111 INFO:teuthology.orchestra.run.smithi063.stdout: pg 3.11 is active+undersized+degraded, acting [4,6] 2024-03-26T21:08:15.111 INFO:teuthology.orchestra.run.smithi063.stdout: pg 3.12 is active+undersized+degraded, acting [0,3] 2024-03-26T21:08:15.111 INFO:teuthology.orchestra.run.smithi063.stdout: pg 3.13 is active+undersized+degraded, acting [4,2] 2024-03-26T21:08:15.111 INFO:teuthology.orchestra.run.smithi063.stdout: pg 3.14 is active+undersized+degraded, acting [4,6] 2024-03-26T21:08:15.111 INFO:teuthology.orchestra.run.smithi063.stdout: pg 3.15 is active+undersized+degraded, acting [3,4] 2024-03-26T21:08:15.111 INFO:teuthology.orchestra.run.smithi063.stdout: pg 3.16 is active+undersized+degraded, acting [5,1] 2024-03-26T21:08:15.111 INFO:teuthology.orchestra.run.smithi063.stdout: pg 3.1b is active+undersized+degraded, acting [0,4] 2024-03-26T21:08:15.111 INFO:teuthology.orchestra.run.smithi063.stdout: pg 4.0 is active+undersized+degraded, acting [3,0] 2024-03-26T21:08:15.111 INFO:teuthology.orchestra.run.smithi063.stdout: pg 4.5 is active+undersized+degraded, acting [6,4] 2024-03-26T21:08:15.111 INFO:teuthology.orchestra.run.smithi063.stdout: pg 4.a is active+undersized+degraded, acting [6,1] 2024-03-26T21:08:15.111 INFO:teuthology.orchestra.run.smithi063.stdout: pg 4.14 is active+undersized+degraded, acting [3,2] 2024-03-26T21:08:16.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:15 smithi174 ceph-mon[53014]: pgmap v295: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 326 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 12 MiB/s wr, 449 op/s; 201/1323 objects degraded (15.193%) 2024-03-26T21:08:16.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:15 smithi174 ceph-mon[53014]: from='client.34458 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:08:16.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:15 smithi174 ceph-mon[53014]: from='client.? 172.21.15.63:0/4294458921' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-26T21:08:16.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:15 smithi063 ceph-mon[66934]: pgmap v295: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 326 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 12 MiB/s wr, 449 op/s; 201/1323 objects degraded (15.193%) 2024-03-26T21:08:16.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:15 smithi063 ceph-mon[66934]: from='client.34458 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:08:16.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:15 smithi063 ceph-mon[66934]: from='client.? 172.21.15.63:0/4294458921' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-26T21:08:17.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:16 smithi174 ceph-mon[53014]: Health check update: Degraded data redundancy: 174/1218 objects degraded (14.286%), 35 pgs degraded (PG_DEGRADED) 2024-03-26T21:08:17.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:16 smithi063 ceph-mon[66934]: Health check update: Degraded data redundancy: 174/1218 objects degraded (14.286%), 35 pgs degraded (PG_DEGRADED) 2024-03-26T21:08:18.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:17 smithi174 ceph-mon[53014]: pgmap v296: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 309 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 11 MiB/s wr, 483 op/s; 174/1218 objects degraded (14.286%) 2024-03-26T21:08:18.048 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:08:17 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:08:17.882+0000 7feba75627c0 -1 osd.7 0 read_superblock omap replica is missing. 2024-03-26T21:08:18.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:17 smithi063 ceph-mon[66934]: pgmap v296: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 309 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 11 MiB/s wr, 483 op/s; 174/1218 objects degraded (14.286%) 2024-03-26T21:08:19.798 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:08:19 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:08:19.337+0000 7feba75627c0 -1 osd.7 146 log_to_monitors true 2024-03-26T21:08:19.798 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:08:19 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:08:19.540+0000 7feb989ba700 -1 osd.7 146 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-26T21:08:19.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:19 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:08:19.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:19 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:08:19.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:19 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:08:19.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:19 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T21:08:19.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:19 smithi174 ceph-mon[53014]: pgmap v297: 97 pgs: 8 active+undersized, 34 active+undersized+degraded, 55 active+clean; 271 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 11 MiB/s wr, 366 op/s; 135/840 objects degraded (16.071%) 2024-03-26T21:08:19.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:19 smithi174 ceph-mon[53014]: pgmap v298: 97 pgs: 8 active+undersized, 34 active+undersized+degraded, 55 active+clean; 271 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 12 MiB/s wr, 346 op/s; 135/840 objects degraded (16.071%) 2024-03-26T21:08:19.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:19 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:08:19.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:19 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:08:19.800 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:19 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:08:19.800 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:19 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:08:19.800 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:19 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:08:19.800 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:19 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:08:19.800 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:19 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:08:19.800 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:19 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:08:19.800 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:19 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-03-26T21:08:19.800 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:19 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-03-26T21:08:19.800 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:19 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-03-26T21:08:19.800 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:19 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-03-26T21:08:19.800 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:19 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-03-26T21:08:19.800 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:19 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-03-26T21:08:19.800 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:19 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-03-26T21:08:19.800 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:19 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-03-26T21:08:19.800 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:19 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-03-26T21:08:19.801 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:19 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-03-26T21:08:19.801 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:19 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-03-26T21:08:19.801 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:19 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-03-26T21:08:19.801 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:19 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-03-26T21:08:19.801 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:19 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-03-26T21:08:19.801 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:19 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-03-26T21:08:19.801 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:19 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-03-26T21:08:19.801 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:19 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-03-26T21:08:19.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:19 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:08:19.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:19 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:08:19.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:19 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:08:19.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:19 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T21:08:19.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:19 smithi063 ceph-mon[66934]: pgmap v297: 97 pgs: 8 active+undersized, 34 active+undersized+degraded, 55 active+clean; 271 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 11 MiB/s wr, 366 op/s; 135/840 objects degraded (16.071%) 2024-03-26T21:08:19.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:19 smithi063 ceph-mon[66934]: pgmap v298: 97 pgs: 8 active+undersized, 34 active+undersized+degraded, 55 active+clean; 271 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 12 MiB/s wr, 346 op/s; 135/840 objects degraded (16.071%) 2024-03-26T21:08:19.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:19 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:08:19.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:19 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:08:19.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:19 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:08:19.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:19 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:08:19.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:19 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:08:19.803 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:19 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:08:19.803 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:19 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:08:19.803 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:19 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:08:19.803 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:19 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-03-26T21:08:19.803 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:19 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-03-26T21:08:19.803 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:19 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-03-26T21:08:19.803 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:19 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-03-26T21:08:19.803 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:19 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-03-26T21:08:19.803 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:19 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-03-26T21:08:19.803 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:19 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-03-26T21:08:19.803 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:19 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-03-26T21:08:19.803 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:19 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-03-26T21:08:19.807 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:19 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-03-26T21:08:19.807 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:19 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-03-26T21:08:19.807 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:19 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-03-26T21:08:19.807 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:19 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-03-26T21:08:19.807 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:19 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-03-26T21:08:19.807 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:19 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-03-26T21:08:19.807 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:19 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-03-26T21:08:19.808 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:19 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-03-26T21:08:20.419 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:20 smithi063 ceph-mon[66934]: Upgrade: Setting container_image for all osd 2024-03-26T21:08:20.419 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:20 smithi063 ceph-mon[66934]: Upgrade: Setting require_osd_release to 19 squid 2024-03-26T21:08:20.419 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:20 smithi063 ceph-mon[66934]: from='osd.7 [v2:172.21.15.174:6824/1210689683,v1:172.21.15.174:6825/1210689683]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-03-26T21:08:20.419 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:20 smithi063 ceph-mon[66934]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-03-26T21:08:20.419 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:20 smithi063 ceph-mon[66934]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-03-26T21:08:20.420 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:20 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-03-26T21:08:20.420 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:20 smithi063 ceph-mon[66934]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-03-26T21:08:20.420 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:20 smithi063 ceph-mon[66934]: osdmap e149: 8 total, 7 up, 8 in 2024-03-26T21:08:20.420 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:20 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi063.tzaznz"]}]: dispatch 2024-03-26T21:08:20.420 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:20 smithi063 ceph-mon[66934]: Upgrade: It appears safe to stop mds.foofs.smithi063.tzaznz 2024-03-26T21:08:20.420 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:20 smithi063 ceph-mon[66934]: from='osd.7 [v2:172.21.15.174:6824/1210689683,v1:172.21.15.174:6825/1210689683]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]: dispatch 2024-03-26T21:08:20.420 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:20 smithi063 ceph-mon[66934]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]: dispatch 2024-03-26T21:08:20.420 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:20 smithi063 ceph-mon[66934]: pgmap v300: 97 pgs: 8 active+undersized, 34 active+undersized+degraded, 55 active+clean; 226 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5.5 KiB/s rd, 5.7 MiB/s wr, 257 op/s; 79/435 objects degraded (18.161%) 2024-03-26T21:08:20.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:20 smithi174 ceph-mon[53014]: Upgrade: Setting container_image for all osd 2024-03-26T21:08:20.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:20 smithi174 ceph-mon[53014]: Upgrade: Setting require_osd_release to 19 squid 2024-03-26T21:08:20.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:20 smithi174 ceph-mon[53014]: from='osd.7 [v2:172.21.15.174:6824/1210689683,v1:172.21.15.174:6825/1210689683]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-03-26T21:08:20.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:20 smithi174 ceph-mon[53014]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-03-26T21:08:20.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:20 smithi174 ceph-mon[53014]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-03-26T21:08:20.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:20 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-03-26T21:08:20.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:20 smithi174 ceph-mon[53014]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-03-26T21:08:20.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:20 smithi174 ceph-mon[53014]: osdmap e149: 8 total, 7 up, 8 in 2024-03-26T21:08:20.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:20 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi063.tzaznz"]}]: dispatch 2024-03-26T21:08:20.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:20 smithi174 ceph-mon[53014]: Upgrade: It appears safe to stop mds.foofs.smithi063.tzaznz 2024-03-26T21:08:20.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:20 smithi174 ceph-mon[53014]: from='osd.7 [v2:172.21.15.174:6824/1210689683,v1:172.21.15.174:6825/1210689683]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]: dispatch 2024-03-26T21:08:20.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:20 smithi174 ceph-mon[53014]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]: dispatch 2024-03-26T21:08:20.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:20 smithi174 ceph-mon[53014]: pgmap v300: 97 pgs: 8 active+undersized, 34 active+undersized+degraded, 55 active+clean; 226 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5.5 KiB/s rd, 5.7 MiB/s wr, 257 op/s; 79/435 objects degraded (18.161%) 2024-03-26T21:08:21.551 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:21 smithi063 ceph-mon[66934]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-26T21:08:21.551 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:21 smithi063 ceph-mon[66934]: osd.7 [v2:172.21.15.174:6824/1210689683,v1:172.21.15.174:6825/1210689683] boot 2024-03-26T21:08:21.551 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:21 smithi063 ceph-mon[66934]: osdmap e150: 8 total, 8 up, 8 in 2024-03-26T21:08:21.551 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:21 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-26T21:08:21.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:21 smithi174 ceph-mon[53014]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-26T21:08:21.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:21 smithi174 ceph-mon[53014]: osd.7 [v2:172.21.15.174:6824/1210689683,v1:172.21.15.174:6825/1210689683] boot 2024-03-26T21:08:21.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:21 smithi174 ceph-mon[53014]: osdmap e150: 8 total, 8 up, 8 in 2024-03-26T21:08:21.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:21 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-26T21:08:22.633 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:22 smithi063 ceph-mon[66934]: osdmap e151: 8 total, 8 up, 8 in 2024-03-26T21:08:22.633 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:22 smithi063 ceph-mon[66934]: Upgrade: Updating mds.foofs.smithi063.tzaznz 2024-03-26T21:08:22.633 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:22 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:08:22.633 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:22 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi063.tzaznz", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-26T21:08:22.633 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:22 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:08:22.633 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:22 smithi063 ceph-mon[66934]: Deploying daemon mds.foofs.smithi063.tzaznz on smithi063 2024-03-26T21:08:22.633 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:22 smithi063 ceph-mon[66934]: pgmap v303: 97 pgs: 7 peering, 3 active+undersized, 26 active+undersized+degraded, 61 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 85 KiB/s wr, 42 op/s; 68/429 objects degraded (15.851%); 0 B/s, 14 objects/s recovering 2024-03-26T21:08:22.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:22 smithi174 ceph-mon[53014]: osdmap e151: 8 total, 8 up, 8 in 2024-03-26T21:08:22.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:22 smithi174 ceph-mon[53014]: Upgrade: Updating mds.foofs.smithi063.tzaznz 2024-03-26T21:08:22.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:22 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:08:22.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:22 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi063.tzaznz", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-26T21:08:22.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:22 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:08:22.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:22 smithi174 ceph-mon[53014]: Deploying daemon mds.foofs.smithi063.tzaznz on smithi063 2024-03-26T21:08:22.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:22 smithi174 ceph-mon[53014]: pgmap v303: 97 pgs: 7 peering, 3 active+undersized, 26 active+undersized+degraded, 61 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 85 KiB/s wr, 42 op/s; 68/429 objects degraded (15.851%); 0 B/s, 14 objects/s recovering 2024-03-26T21:08:23.747 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:23 smithi063 ceph-mon[66934]: osdmap e152: 8 total, 8 up, 8 in 2024-03-26T21:08:23.747 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:23 smithi063 ceph-mon[66934]: Health check update: Degraded data redundancy: 68/429 objects degraded (15.851%), 26 pgs degraded (PG_DEGRADED) 2024-03-26T21:08:23.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:23 smithi174 ceph-mon[53014]: osdmap e152: 8 total, 8 up, 8 in 2024-03-26T21:08:23.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:23 smithi174 ceph-mon[53014]: Health check update: Degraded data redundancy: 68/429 objects degraded (15.851%), 26 pgs degraded (PG_DEGRADED) 2024-03-26T21:08:24.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:24 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-mon-smithi063[66930]: 2024-03-26T21:08:24.052+0000 7f97cf0bd700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-26T21:08:24.302 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 21:08:24 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-1[75620]: 2024-03-26T21:08:24.073+0000 7f396ecb3700 -1 bdev(0x5597988be000 /var/lib/ceph/osd/ceph-1/block) aio_submit retries 5 2024-03-26T21:08:24.302 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 21:08:24 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-1[75620]: 2024-03-26T21:08:24.073+0000 7f3969ca9700 -1 bdev(0x5597988be000 /var/lib/ceph/osd/ceph-1/block) aio_submit retries 12 2024-03-26T21:08:24.302 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 21:08:24 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-1[75620]: 2024-03-26T21:08:24.073+0000 7f396e4b2700 -1 bdev(0x5597988be000 /var/lib/ceph/osd/ceph-1/block) aio_submit retries 13 2024-03-26T21:08:25.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:24 smithi174 ceph-mon[53014]: osdmap e153: 8 total, 8 up, 8 in 2024-03-26T21:08:25.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:24 smithi174 ceph-mon[53014]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-03-26T21:08:25.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:24 smithi174 ceph-mon[53014]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-26T21:08:25.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:24 smithi174 ceph-mon[53014]: osdmap e154: 8 total, 8 up, 8 in 2024-03-26T21:08:25.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:24 smithi174 ceph-mon[53014]: Standby daemon mds.foofs.smithi174.gitdvk assigned to filesystem foofs as rank 0 2024-03-26T21:08:25.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:24 smithi174 ceph-mon[53014]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-03-26T21:08:25.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:24 smithi174 ceph-mon[53014]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-03-26T21:08:25.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:24 smithi174 ceph-mon[53014]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-03-26T21:08:25.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:24 smithi174 ceph-mon[53014]: fsmap foofs:1/1 {0=foofs.smithi174.gitdvk=up:replay} 2024-03-26T21:08:25.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:24 smithi174 ceph-mon[53014]: pgmap v307: 97 pgs: 1 activating+undersized+degraded+remapped, 7 peering, 1 active+undersized, 20 active+undersized+degraded, 68 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 270 B/s rd, 0 B/s wr, 0 op/s; 58/429 objects degraded (13.520%); 0 B/s, 39 objects/s recovering 2024-03-26T21:08:25.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:24 smithi063 ceph-mon[66934]: osdmap e153: 8 total, 8 up, 8 in 2024-03-26T21:08:25.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:24 smithi063 ceph-mon[66934]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-03-26T21:08:25.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:24 smithi063 ceph-mon[66934]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-26T21:08:25.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:24 smithi063 ceph-mon[66934]: osdmap e154: 8 total, 8 up, 8 in 2024-03-26T21:08:25.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:24 smithi063 ceph-mon[66934]: Standby daemon mds.foofs.smithi174.gitdvk assigned to filesystem foofs as rank 0 2024-03-26T21:08:25.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:24 smithi063 ceph-mon[66934]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-03-26T21:08:25.052 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:24 smithi063 ceph-mon[66934]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-03-26T21:08:25.052 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:24 smithi063 ceph-mon[66934]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-03-26T21:08:25.052 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:24 smithi063 ceph-mon[66934]: fsmap foofs:1/1 {0=foofs.smithi174.gitdvk=up:replay} 2024-03-26T21:08:25.052 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:24 smithi063 ceph-mon[66934]: pgmap v307: 97 pgs: 1 activating+undersized+degraded+remapped, 7 peering, 1 active+undersized, 20 active+undersized+degraded, 68 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 270 B/s rd, 0 B/s wr, 0 op/s; 58/429 objects degraded (13.520%); 0 B/s, 39 objects/s recovering 2024-03-26T21:08:27.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:26 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:08:27.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:26 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:08:27.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:26 smithi174 ceph-mon[53014]: pgmap v308: 97 pgs: 1 activating+undersized+degraded+remapped, 7 peering, 89 active+clean; 224 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6.7 MiB/s rd, 2.1 KiB/s wr, 5 op/s; 1/426 objects degraded (0.235%); 856 KiB/s, 65 objects/s recovering 2024-03-26T21:08:27.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:26 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi063.tzaznz"}]: dispatch 2024-03-26T21:08:27.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:26 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:08:27.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:26 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:08:27.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:26 smithi063 ceph-mon[66934]: pgmap v308: 97 pgs: 1 activating+undersized+degraded+remapped, 7 peering, 89 active+clean; 224 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6.7 MiB/s rd, 2.1 KiB/s wr, 5 op/s; 1/426 objects degraded (0.235%); 856 KiB/s, 65 objects/s recovering 2024-03-26T21:08:27.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:26 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi063.tzaznz"}]: dispatch 2024-03-26T21:08:28.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:28 smithi174 ceph-mon[53014]: pgmap v309: 97 pgs: 97 active+clean; 224 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 1.7 KiB/s wr, 5 op/s; 1.3 MiB/s, 51 objects/s recovering 2024-03-26T21:08:28.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:28 smithi174 ceph-mon[53014]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/426 objects degraded (0.235%), 1 pg degraded) 2024-03-26T21:08:28.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:28 smithi063 ceph-mon[66934]: pgmap v309: 97 pgs: 97 active+clean; 224 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 1.7 KiB/s wr, 5 op/s; 1.3 MiB/s, 51 objects/s recovering 2024-03-26T21:08:28.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:28 smithi063 ceph-mon[66934]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/426 objects degraded (0.235%), 1 pg degraded) 2024-03-26T21:08:30.643 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:30 smithi174 ceph-mon[53014]: reconnect by client.24365 172.21.15.174:0/4163182094 after 0 2024-03-26T21:08:30.643 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:30 smithi174 ceph-mon[53014]: reconnect by client.14530 172.21.15.63:0/4061700709 after 0 2024-03-26T21:08:30.643 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:30 smithi174 ceph-mon[53014]: mds.? [v2:172.21.15.174:6832/2611248035,v1:172.21.15.174:6833/2611248035] up:reconnect 2024-03-26T21:08:30.643 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:30 smithi174 ceph-mon[53014]: fsmap foofs:1/1 {0=foofs.smithi174.gitdvk=up:reconnect} 2024-03-26T21:08:30.643 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:30 smithi174 ceph-mon[53014]: pgmap v310: 97 pgs: 97 active+clean; 224 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 1.3 KiB/s wr, 8 op/s; 1.0 MiB/s, 39 objects/s recovering 2024-03-26T21:08:30.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:30 smithi063 ceph-mon[66934]: reconnect by client.24365 172.21.15.174:0/4163182094 after 0 2024-03-26T21:08:30.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:30 smithi063 ceph-mon[66934]: reconnect by client.14530 172.21.15.63:0/4061700709 after 0 2024-03-26T21:08:30.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:30 smithi063 ceph-mon[66934]: mds.? [v2:172.21.15.174:6832/2611248035,v1:172.21.15.174:6833/2611248035] up:reconnect 2024-03-26T21:08:30.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:30 smithi063 ceph-mon[66934]: fsmap foofs:1/1 {0=foofs.smithi174.gitdvk=up:reconnect} 2024-03-26T21:08:30.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:30 smithi063 ceph-mon[66934]: pgmap v310: 97 pgs: 97 active+clean; 224 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 1.3 KiB/s wr, 8 op/s; 1.0 MiB/s, 39 objects/s recovering 2024-03-26T21:08:31.659 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:31 smithi174 ceph-mon[53014]: mds.? [v2:172.21.15.174:6832/2611248035,v1:172.21.15.174:6833/2611248035] up:rejoin 2024-03-26T21:08:31.660 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:31 smithi174 ceph-mon[53014]: fsmap foofs:1/1 {0=foofs.smithi174.gitdvk=up:rejoin} 2024-03-26T21:08:31.660 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:31 smithi174 ceph-mon[53014]: daemon mds.foofs.smithi174.gitdvk is now active in filesystem foofs as rank 0 2024-03-26T21:08:31.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:31 smithi063 ceph-mon[66934]: mds.? [v2:172.21.15.174:6832/2611248035,v1:172.21.15.174:6833/2611248035] up:rejoin 2024-03-26T21:08:31.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:31 smithi063 ceph-mon[66934]: fsmap foofs:1/1 {0=foofs.smithi174.gitdvk=up:rejoin} 2024-03-26T21:08:31.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:31 smithi063 ceph-mon[66934]: daemon mds.foofs.smithi174.gitdvk is now active in filesystem foofs as rank 0 2024-03-26T21:08:32.551 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:32 smithi063 ceph-mon[66934]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-03-26T21:08:32.551 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:32 smithi063 ceph-mon[66934]: mds.? [v2:172.21.15.174:6832/2611248035,v1:172.21.15.174:6833/2611248035] up:active 2024-03-26T21:08:32.551 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:32 smithi063 ceph-mon[66934]: fsmap foofs:1 {0=foofs.smithi174.gitdvk=up:active} 2024-03-26T21:08:32.551 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:32 smithi063 ceph-mon[66934]: pgmap v311: 97 pgs: 97 active+clean; 224 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 26 MiB/s rd, 1.2 KiB/s wr, 9 op/s; 963 KiB/s, 32 objects/s recovering 2024-03-26T21:08:32.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:32 smithi174 ceph-mon[53014]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-03-26T21:08:32.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:32 smithi174 ceph-mon[53014]: mds.? [v2:172.21.15.174:6832/2611248035,v1:172.21.15.174:6833/2611248035] up:active 2024-03-26T21:08:32.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:32 smithi174 ceph-mon[53014]: fsmap foofs:1 {0=foofs.smithi174.gitdvk=up:active} 2024-03-26T21:08:32.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:32 smithi174 ceph-mon[53014]: pgmap v311: 97 pgs: 97 active+clean; 224 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 26 MiB/s rd, 1.2 KiB/s wr, 9 op/s; 963 KiB/s, 32 objects/s recovering 2024-03-26T21:08:33.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:33 smithi174 ceph-mon[53014]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-03-26T21:08:33.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:33 smithi174 ceph-mon[53014]: mds.? [v2:172.21.15.63:6834/3891894564,v1:172.21.15.63:6835/3891894564] up:boot 2024-03-26T21:08:33.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:33 smithi174 ceph-mon[53014]: fsmap foofs:1 {0=foofs.smithi174.gitdvk=up:active} 1 up:standby 2024-03-26T21:08:33.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:33 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi063.tzaznz"}]: dispatch 2024-03-26T21:08:33.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:33 smithi063 ceph-mon[66934]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-03-26T21:08:33.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:33 smithi063 ceph-mon[66934]: mds.? [v2:172.21.15.63:6834/3891894564,v1:172.21.15.63:6835/3891894564] up:boot 2024-03-26T21:08:33.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:33 smithi063 ceph-mon[66934]: fsmap foofs:1 {0=foofs.smithi174.gitdvk=up:active} 1 up:standby 2024-03-26T21:08:33.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:33 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi063.tzaznz"}]: dispatch 2024-03-26T21:08:34.775 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:34 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:08:34.775 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:34 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:08:34.775 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:34 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:08:34.775 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:34 smithi063 ceph-mon[66934]: pgmap v312: 97 pgs: 97 active+clean; 224 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 999 B/s wr, 8 op/s; 799 KiB/s, 27 objects/s recovering 2024-03-26T21:08:34.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:34 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:08:34.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:34 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:08:34.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:34 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:08:34.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:34 smithi174 ceph-mon[53014]: pgmap v312: 97 pgs: 97 active+clean; 224 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 999 B/s wr, 8 op/s; 799 KiB/s, 27 objects/s recovering 2024-03-26T21:08:37.260 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:36 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:08:37.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:36 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:08:38.271 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:38 smithi063 ceph-mon[66934]: pgmap v313: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 6.8 KiB/s wr, 12 op/s; 683 KiB/s, 18 objects/s recovering 2024-03-26T21:08:38.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:38 smithi174 ceph-mon[53014]: pgmap v313: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 6.8 KiB/s wr, 12 op/s; 683 KiB/s, 18 objects/s recovering 2024-03-26T21:08:39.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:39 smithi174 ceph-mon[53014]: pgmap v314: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 6.0 KiB/s wr, 10 op/s; 341 KiB/s, 4 objects/s recovering 2024-03-26T21:08:39.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:39 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:08:39.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:39 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:08:39.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:39 smithi063 ceph-mon[66934]: pgmap v314: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 6.0 KiB/s wr, 10 op/s; 341 KiB/s, 4 objects/s recovering 2024-03-26T21:08:39.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:39 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:08:39.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:39 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:08:40.951 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:40 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:08:40.952 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:40 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:08:40.952 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:40 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:08:40.952 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:40 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:08:41.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:40 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:08:41.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:40 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:08:41.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:40 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:08:41.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:40 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:08:41.984 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:41 smithi063 ceph-mon[66934]: pgmap v315: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 7.2 KiB/s wr, 9 op/s 2024-03-26T21:08:42.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:41 smithi174 ceph-mon[53014]: pgmap v315: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 7.2 KiB/s wr, 9 op/s 2024-03-26T21:08:43.954 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:43 smithi063 ceph-mon[66934]: pgmap v316: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 7.2 KiB/s wr, 7 op/s 2024-03-26T21:08:44.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:43 smithi174 ceph-mon[53014]: pgmap v316: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 7.2 KiB/s wr, 7 op/s 2024-03-26T21:08:45.183 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:45 smithi174 ceph-mon[53014]: Detected new or changed devices on smithi063 2024-03-26T21:08:45.184 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:45 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:08:45.184 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:45 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:08:45.184 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:45 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:08:45.184 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:45 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T21:08:45.184 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:45 smithi174 ceph-mon[53014]: pgmap v317: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 353 KiB/s rd, 7.4 KiB/s wr, 6 op/s 2024-03-26T21:08:45.184 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:45 smithi174 ceph-mon[53014]: pgmap v318: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3.4 KiB/s rd, 9.0 KiB/s wr, 7 op/s 2024-03-26T21:08:45.184 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:45 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:08:45.184 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:45 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:08:45.184 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:45 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:08:45.184 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:45 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:08:45.184 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:45 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:08:45.184 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:45 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:08:45.185 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:45 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:08:45.185 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:45 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi174.gitdvk"]}]: dispatch 2024-03-26T21:08:45.185 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:45 smithi174 ceph-mon[53014]: Upgrade: It appears safe to stop mds.foofs.smithi174.gitdvk 2024-03-26T21:08:45.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:45 smithi063 ceph-mon[66934]: Detected new or changed devices on smithi063 2024-03-26T21:08:45.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:45 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:08:45.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:45 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:08:45.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:45 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:08:45.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:45 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T21:08:45.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:45 smithi063 ceph-mon[66934]: pgmap v317: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 353 KiB/s rd, 7.4 KiB/s wr, 6 op/s 2024-03-26T21:08:45.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:45 smithi063 ceph-mon[66934]: pgmap v318: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3.4 KiB/s rd, 9.0 KiB/s wr, 7 op/s 2024-03-26T21:08:45.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:45 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:08:45.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:45 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:08:45.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:45 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:08:45.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:45 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:08:45.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:45 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:08:45.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:45 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:08:45.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:45 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:08:45.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:45 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi174.gitdvk"]}]: dispatch 2024-03-26T21:08:45.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:45 smithi063 ceph-mon[66934]: Upgrade: It appears safe to stop mds.foofs.smithi174.gitdvk 2024-03-26T21:08:45.483 INFO:teuthology.orchestra.run.smithi063.stdout:true 2024-03-26T21:08:46.171 INFO:teuthology.orchestra.run.smithi063.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-26T21:08:46.171 INFO:teuthology.orchestra.run.smithi063.stdout:alertmanager.smithi063 smithi063 *:9093,9094 running (10m) 7s ago 19m 24.7M - 0.23.0 ba2b418f427c 6e160c94aac3 2024-03-26T21:08:46.171 INFO:teuthology.orchestra.run.smithi063.stdout:crash.smithi063 smithi063 running (6m) 7s ago 19m 6665k - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 cf1096d84270 2024-03-26T21:08:46.171 INFO:teuthology.orchestra.run.smithi063.stdout:crash.smithi174 smithi174 running (5m) 32s ago 18m 6665k - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 cb5371a92684 2024-03-26T21:08:46.171 INFO:teuthology.orchestra.run.smithi063.stdout:grafana.smithi063 smithi063 *:3000 error 7s ago 18m - - 2024-03-26T21:08:46.171 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foofs.smithi063.tzaznz smithi063 running (13s) 7s ago 13m 11.4M - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 ef76ec94effe 2024-03-26T21:08:46.171 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foofs.smithi174.gitdvk smithi174 running (13m) 32s ago 13m 14.4M - 17.2.0 e1d6a67b021e 4339a5d98d1d 2024-03-26T21:08:46.171 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.smithi063.lxhujt smithi063 *:8443,9283,8765 running (8m) 7s ago 20m 514M - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 6d9d10330a90 2024-03-26T21:08:46.171 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.smithi174.sexoin smithi174 *:8443,9283,8765 running (8m) 32s ago 18m 453M - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 d7546233672d 2024-03-26T21:08:46.172 INFO:teuthology.orchestra.run.smithi063.stdout:mon.smithi063 smithi063 running (7m) 7s ago 20m 62.1M 2048M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 c644200344f8 2024-03-26T21:08:46.172 INFO:teuthology.orchestra.run.smithi063.stdout:mon.smithi174 smithi174 running (7m) 32s ago 18m 47.9M 2048M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 1a6d442692e7 2024-03-26T21:08:46.172 INFO:teuthology.orchestra.run.smithi063.stdout:nfs.foo.0.0.smithi063.fgmpwa smithi063 *:2049 running (12m) 7s ago 12m 271M - 4.0 e1d6a67b021e 3d6c0dd9ce51 2024-03-26T21:08:46.172 INFO:teuthology.orchestra.run.smithi063.stdout:nfs.foo.1.0.smithi174.pytptt smithi174 *:2049 running (12m) 32s ago 12m 86.9M - 4.0 e1d6a67b021e f220dbb449a3 2024-03-26T21:08:46.172 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.smithi063 smithi063 *:9100 running (9m) 7s ago 18m 19.8M - 1.3.1 1dbe0e931976 f52f41fd14c0 2024-03-26T21:08:46.172 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.smithi174 smithi174 *:9100 running (9m) 32s ago 18m 18.4M - 1.3.1 1dbe0e931976 0c027f39a4c8 2024-03-26T21:08:46.172 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0 smithi063 running (5m) 7s ago 17m 416M 1936M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 2d4b3fa825cf 2024-03-26T21:08:46.172 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1 smithi063 running (4m) 7s ago 17m 370M 1936M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 d41fbe507197 2024-03-26T21:08:46.172 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2 smithi063 running (3m) 7s ago 16m 347M 1936M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 f123eeabd3fa 2024-03-26T21:08:46.172 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3 smithi063 running (3m) 7s ago 16m 423M 1936M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 953816eafe87 2024-03-26T21:08:46.172 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4 smithi174 running (2m) 32s ago 15m 349M 2704M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 8aa35bff6cea 2024-03-26T21:08:46.172 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5 smithi174 running (114s) 32s ago 15m 307M 2704M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 25dde01e0676 2024-03-26T21:08:46.172 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6 smithi174 running (76s) 32s ago 15m 286M 2704M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 95fa1b77701b 2024-03-26T21:08:46.172 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7 smithi174 running (37s) 32s ago 14m 12.7M 2704M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 cb89961bc930 2024-03-26T21:08:46.172 INFO:teuthology.orchestra.run.smithi063.stdout:prometheus.smithi063 smithi063 *:9095 running (8m) 7s ago 17m 76.2M - 2.33.4 514e6a882f6e 567eef202627 2024-03-26T21:08:46.587 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-03-26T21:08:46.587 INFO:teuthology.orchestra.run.smithi063.stdout: "mon": { 2024-03-26T21:08:46.587 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 2 2024-03-26T21:08:46.587 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:08:46.588 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr": { 2024-03-26T21:08:46.588 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 2 2024-03-26T21:08:46.588 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:08:46.588 INFO:teuthology.orchestra.run.smithi063.stdout: "osd": { 2024-03-26T21:08:46.588 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 8 2024-03-26T21:08:46.588 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:08:46.588 INFO:teuthology.orchestra.run.smithi063.stdout: "mds": { 2024-03-26T21:08:46.588 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-03-26T21:08:46.588 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 1 2024-03-26T21:08:46.588 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:08:46.588 INFO:teuthology.orchestra.run.smithi063.stdout: "overall": { 2024-03-26T21:08:46.588 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-03-26T21:08:46.588 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 13 2024-03-26T21:08:46.588 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-03-26T21:08:46.588 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-03-26T21:08:46.936 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-03-26T21:08:46.937 INFO:teuthology.orchestra.run.smithi063.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d", 2024-03-26T21:08:46.937 INFO:teuthology.orchestra.run.smithi063.stdout: "in_progress": true, 2024-03-26T21:08:46.937 INFO:teuthology.orchestra.run.smithi063.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-26T21:08:46.937 INFO:teuthology.orchestra.run.smithi063.stdout: "services_complete": [ 2024-03-26T21:08:46.937 INFO:teuthology.orchestra.run.smithi063.stdout: "osd", 2024-03-26T21:08:46.937 INFO:teuthology.orchestra.run.smithi063.stdout: "crash", 2024-03-26T21:08:46.937 INFO:teuthology.orchestra.run.smithi063.stdout: "mon", 2024-03-26T21:08:46.937 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr" 2024-03-26T21:08:46.937 INFO:teuthology.orchestra.run.smithi063.stdout: ], 2024-03-26T21:08:46.937 INFO:teuthology.orchestra.run.smithi063.stdout: "progress": "15/23 daemons upgraded", 2024-03-26T21:08:46.937 INFO:teuthology.orchestra.run.smithi063.stdout: "message": "Currently upgrading mds daemons", 2024-03-26T21:08:46.937 INFO:teuthology.orchestra.run.smithi063.stdout: "is_paused": false 2024-03-26T21:08:46.938 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-03-26T21:08:47.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:46 smithi174 ceph-mon[53014]: from='client.34472 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:08:47.050 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:46 smithi174 ceph-mon[53014]: from='client.34476 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:08:47.050 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:46 smithi174 ceph-mon[53014]: Upgrade: Updating mds.foofs.smithi174.gitdvk 2024-03-26T21:08:47.050 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:46 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:08:47.050 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:46 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi174.gitdvk", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-26T21:08:47.050 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:46 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:08:47.050 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:46 smithi174 ceph-mon[53014]: Deploying daemon mds.foofs.smithi174.gitdvk on smithi174 2024-03-26T21:08:47.050 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:46 smithi174 ceph-mon[53014]: pgmap v319: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 211 B/s rd, 1.8 KiB/s wr, 0 op/s 2024-03-26T21:08:47.050 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:46 smithi174 ceph-mon[53014]: from='client.34480 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:08:47.050 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:46 smithi174 ceph-mon[53014]: from='client.? 172.21.15.63:0/2534094727' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:08:47.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:46 smithi063 ceph-mon[66934]: from='client.34472 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:08:47.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:46 smithi063 ceph-mon[66934]: from='client.34476 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:08:47.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:46 smithi063 ceph-mon[66934]: Upgrade: Updating mds.foofs.smithi174.gitdvk 2024-03-26T21:08:47.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:46 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:08:47.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:46 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi174.gitdvk", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-26T21:08:47.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:46 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:08:47.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:46 smithi063 ceph-mon[66934]: Deploying daemon mds.foofs.smithi174.gitdvk on smithi174 2024-03-26T21:08:47.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:46 smithi063 ceph-mon[66934]: pgmap v319: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 211 B/s rd, 1.8 KiB/s wr, 0 op/s 2024-03-26T21:08:47.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:46 smithi063 ceph-mon[66934]: from='client.34480 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:08:47.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:46 smithi063 ceph-mon[66934]: from='client.? 172.21.15.63:0/2534094727' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:08:47.349 INFO:teuthology.orchestra.run.smithi063.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-26T21:08:47.349 INFO:teuthology.orchestra.run.smithi063.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-26T21:08:47.349 INFO:teuthology.orchestra.run.smithi063.stdout: daemon grafana.smithi063 on smithi063 is in error state 2024-03-26T21:08:47.940 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:47 smithi174 ceph-mon[53014]: from='client.34488 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:08:47.940 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:47 smithi174 ceph-mon[53014]: from='client.? 172.21.15.63:0/1919569563' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-26T21:08:48.298 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:08:48 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-5[60157]: 2024-03-26T21:08:48.074+0000 7f27f9325700 -1 bdev(0x562638396000 /var/lib/ceph/osd/ceph-5/block) aio_submit retries 2 2024-03-26T21:08:48.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:47 smithi063 ceph-mon[66934]: from='client.34488 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:08:48.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:47 smithi063 ceph-mon[66934]: from='client.? 172.21.15.63:0/1919569563' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-26T21:08:48.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:47 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-mon-smithi063[66930]: 2024-03-26T21:08:47.949+0000 7f97cf0bd700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-26T21:08:48.302 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 21:08:48 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-1[75620]: 2024-03-26T21:08:48.067+0000 7f396a4aa700 -1 bdev(0x5597988be000 /var/lib/ceph/osd/ceph-1/block) aio_submit retries 1 2024-03-26T21:08:48.302 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 21:08:48 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-1[75620]: 2024-03-26T21:08:48.073+0000 7f396acab700 -1 bdev(0x5597988be000 /var/lib/ceph/osd/ceph-1/block) aio_submit retries 3 2024-03-26T21:08:48.302 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 21:08:48 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-1[75620]: 2024-03-26T21:08:48.073+0000 7f396e4b2700 -1 bdev(0x5597988be000 /var/lib/ceph/osd/ceph-1/block) aio_submit retries 10 2024-03-26T21:08:48.302 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 21:08:48 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-1[75620]: 2024-03-26T21:08:48.075+0000 7f396dcb1700 -1 bdev(0x5597988be000 /var/lib/ceph/osd/ceph-1/block) aio_submit retries 19 2024-03-26T21:08:49.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:48 smithi174 ceph-mon[53014]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-03-26T21:08:49.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:48 smithi174 ceph-mon[53014]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-26T21:08:49.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:48 smithi174 ceph-mon[53014]: osdmap e155: 8 total, 8 up, 8 in 2024-03-26T21:08:49.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:48 smithi174 ceph-mon[53014]: Standby daemon mds.foofs.smithi063.tzaznz assigned to filesystem foofs as rank 0 2024-03-26T21:08:49.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:48 smithi174 ceph-mon[53014]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-03-26T21:08:49.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:48 smithi174 ceph-mon[53014]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-03-26T21:08:49.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:48 smithi174 ceph-mon[53014]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-03-26T21:08:49.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:48 smithi174 ceph-mon[53014]: fsmap foofs:1/1 {0=foofs.smithi063.tzaznz=up:replay} 2024-03-26T21:08:49.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:48 smithi174 ceph-mon[53014]: pgmap v321: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 265 B/s rd, 265 B/s wr, 0 op/s 2024-03-26T21:08:49.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:48 smithi063 ceph-mon[66934]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-03-26T21:08:49.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:48 smithi063 ceph-mon[66934]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-26T21:08:49.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:48 smithi063 ceph-mon[66934]: osdmap e155: 8 total, 8 up, 8 in 2024-03-26T21:08:49.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:48 smithi063 ceph-mon[66934]: Standby daemon mds.foofs.smithi063.tzaznz assigned to filesystem foofs as rank 0 2024-03-26T21:08:49.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:48 smithi063 ceph-mon[66934]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-03-26T21:08:49.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:48 smithi063 ceph-mon[66934]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-03-26T21:08:49.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:48 smithi063 ceph-mon[66934]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-03-26T21:08:49.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:48 smithi063 ceph-mon[66934]: fsmap foofs:1/1 {0=foofs.smithi063.tzaznz=up:replay} 2024-03-26T21:08:49.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:48 smithi063 ceph-mon[66934]: pgmap v321: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 265 B/s rd, 265 B/s wr, 0 op/s 2024-03-26T21:08:51.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:50 smithi174 ceph-mon[53014]: reconnect by client.14530 172.21.15.63:0/4061700709 after 0 2024-03-26T21:08:51.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:50 smithi174 ceph-mon[53014]: reconnect by client.24365 172.21.15.174:0/4163182094 after 0 2024-03-26T21:08:51.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:50 smithi174 ceph-mon[53014]: mds.? [v2:172.21.15.63:6834/3891894564,v1:172.21.15.63:6835/3891894564] up:reconnect 2024-03-26T21:08:51.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:50 smithi174 ceph-mon[53014]: fsmap foofs:1/1 {0=foofs.smithi063.tzaznz=up:reconnect} 2024-03-26T21:08:51.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:50 smithi174 ceph-mon[53014]: pgmap v322: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 265 B/s wr, 6 op/s 2024-03-26T21:08:51.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:50 smithi063 ceph-mon[66934]: reconnect by client.14530 172.21.15.63:0/4061700709 after 0 2024-03-26T21:08:51.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:50 smithi063 ceph-mon[66934]: reconnect by client.24365 172.21.15.174:0/4163182094 after 0 2024-03-26T21:08:51.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:50 smithi063 ceph-mon[66934]: mds.? [v2:172.21.15.63:6834/3891894564,v1:172.21.15.63:6835/3891894564] up:reconnect 2024-03-26T21:08:51.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:50 smithi063 ceph-mon[66934]: fsmap foofs:1/1 {0=foofs.smithi063.tzaznz=up:reconnect} 2024-03-26T21:08:51.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:50 smithi063 ceph-mon[66934]: pgmap v322: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 265 B/s wr, 6 op/s 2024-03-26T21:08:52.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:51 smithi174 ceph-mon[53014]: mds.? [v2:172.21.15.63:6834/3891894564,v1:172.21.15.63:6835/3891894564] up:rejoin 2024-03-26T21:08:52.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:51 smithi174 ceph-mon[53014]: fsmap foofs:1/1 {0=foofs.smithi063.tzaznz=up:rejoin} 2024-03-26T21:08:52.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:51 smithi174 ceph-mon[53014]: daemon mds.foofs.smithi063.tzaznz is now active in filesystem foofs as rank 0 2024-03-26T21:08:52.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:51 smithi063 ceph-mon[66934]: mds.? [v2:172.21.15.63:6834/3891894564,v1:172.21.15.63:6835/3891894564] up:rejoin 2024-03-26T21:08:52.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:51 smithi063 ceph-mon[66934]: fsmap foofs:1/1 {0=foofs.smithi063.tzaznz=up:rejoin} 2024-03-26T21:08:52.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:51 smithi063 ceph-mon[66934]: daemon mds.foofs.smithi063.tzaznz is now active in filesystem foofs as rank 0 2024-03-26T21:08:53.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:52 smithi174 ceph-mon[53014]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-03-26T21:08:53.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:52 smithi174 ceph-mon[53014]: mds.? [v2:172.21.15.63:6834/3891894564,v1:172.21.15.63:6835/3891894564] up:active 2024-03-26T21:08:53.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:52 smithi174 ceph-mon[53014]: fsmap foofs:1 {0=foofs.smithi063.tzaznz=up:active} 2024-03-26T21:08:53.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:52 smithi174 ceph-mon[53014]: pgmap v323: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 255 B/s wr, 7 op/s 2024-03-26T21:08:53.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:52 smithi063 ceph-mon[66934]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-03-26T21:08:53.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:52 smithi063 ceph-mon[66934]: mds.? [v2:172.21.15.63:6834/3891894564,v1:172.21.15.63:6835/3891894564] up:active 2024-03-26T21:08:53.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:52 smithi063 ceph-mon[66934]: fsmap foofs:1 {0=foofs.smithi063.tzaznz=up:active} 2024-03-26T21:08:53.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:52 smithi063 ceph-mon[66934]: pgmap v323: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 255 B/s wr, 7 op/s 2024-03-26T21:08:55.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:55 smithi174 ceph-mon[53014]: pgmap v324: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 204 B/s wr, 7 op/s 2024-03-26T21:08:55.551 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:55 smithi063 ceph-mon[66934]: pgmap v324: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 204 B/s wr, 7 op/s 2024-03-26T21:08:56.965 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:56 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:08:56.965 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:56 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:08:56.965 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:56 smithi174 ceph-mon[53014]: pgmap v325: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 7.3 KiB/s wr, 14 op/s 2024-03-26T21:08:57.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:56 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:08:57.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:56 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:08:57.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:56 smithi063 ceph-mon[66934]: pgmap v325: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 7.3 KiB/s wr, 14 op/s 2024-03-26T21:08:59.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:59 smithi174 ceph-mon[53014]: pgmap v326: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 7.3 KiB/s wr, 14 op/s 2024-03-26T21:08:59.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:59 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:08:59.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:59 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:08:59.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:59 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:08:59.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:59 smithi174 ceph-mon[53014]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-03-26T21:08:59.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:59 smithi174 ceph-mon[53014]: mds.? [v2:172.21.15.174:6832/1039757756,v1:172.21.15.174:6833/1039757756] up:boot 2024-03-26T21:08:59.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:59 smithi174 ceph-mon[53014]: fsmap foofs:1 {0=foofs.smithi063.tzaznz=up:active} 1 up:standby 2024-03-26T21:08:59.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:08:59 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi174.gitdvk"}]: dispatch 2024-03-26T21:08:59.551 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:59 smithi063 ceph-mon[66934]: pgmap v326: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 7.3 KiB/s wr, 14 op/s 2024-03-26T21:08:59.551 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:59 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:08:59.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:59 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:08:59.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:59 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:08:59.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:59 smithi063 ceph-mon[66934]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-03-26T21:08:59.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:59 smithi063 ceph-mon[66934]: mds.? [v2:172.21.15.174:6832/1039757756,v1:172.21.15.174:6833/1039757756] up:boot 2024-03-26T21:08:59.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:59 smithi063 ceph-mon[66934]: fsmap foofs:1 {0=foofs.smithi063.tzaznz=up:active} 1 up:standby 2024-03-26T21:08:59.552 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:08:59 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi174.gitdvk"}]: dispatch 2024-03-26T21:09:01.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:01 smithi174 ceph-mon[53014]: pgmap v327: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 7.3 KiB/s wr, 12 op/s 2024-03-26T21:09:01.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:01 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:01.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:01 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:01.551 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:01 smithi063 ceph-mon[66934]: pgmap v327: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 7.3 KiB/s wr, 12 op/s 2024-03-26T21:09:01.551 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:01 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:01.551 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:01 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:03.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:03 smithi174 ceph-mon[53014]: pgmap v328: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5.7 MiB/s rd, 7.3 KiB/s wr, 8 op/s 2024-03-26T21:09:03.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:03 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:03.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:03 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:03.551 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:03 smithi063 ceph-mon[66934]: pgmap v328: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5.7 MiB/s rd, 7.3 KiB/s wr, 8 op/s 2024-03-26T21:09:03.551 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:03 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:03.551 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:03 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:04.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:04 smithi174 ceph-mon[53014]: pgmap v329: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.7 MiB/s rd, 7.3 KiB/s wr, 7 op/s 2024-03-26T21:09:04.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:04 smithi063 ceph-mon[66934]: pgmap v329: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.7 MiB/s rd, 7.3 KiB/s wr, 7 op/s 2024-03-26T21:09:06.890 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:06 smithi063 ceph-mon[66934]: Detected new or changed devices on smithi174 2024-03-26T21:09:06.890 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:06 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:06.890 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:06 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:06.890 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:06 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:09:06.890 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:06 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T21:09:06.890 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:06 smithi063 ceph-mon[66934]: pgmap v330: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.8 KiB/s rd, 7.7 KiB/s wr, 6 op/s 2024-03-26T21:09:06.890 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:06 smithi063 ceph-mon[66934]: pgmap v331: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 209 B/s rd, 1.7 KiB/s wr, 0 op/s 2024-03-26T21:09:06.890 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:06 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:06.890 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:06 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:06.891 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:06 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:09:06.891 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:06 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:09:06.891 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:06 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:09:06.891 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:06 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:09:06.891 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:06 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:09:06.891 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:06 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:09:06.891 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:06 smithi063 ceph-mon[66934]: Upgrade: Setting container_image for all mds 2024-03-26T21:09:06.891 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:06 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:06.891 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:06 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi063.tzaznz"}]: dispatch 2024-03-26T21:09:06.891 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:06 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi063.tzaznz"}]': finished 2024-03-26T21:09:06.891 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:06 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi174.gitdvk"}]: dispatch 2024-03-26T21:09:06.891 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:06 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi174.gitdvk"}]': finished 2024-03-26T21:09:06.891 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:06 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:09:06.891 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:06 smithi063 ceph-mon[66934]: Upgrade: Setting container_image for all rgw 2024-03-26T21:09:06.891 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:06 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:06.892 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:06 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:09:06.892 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:06 smithi063 ceph-mon[66934]: Upgrade: Setting container_image for all rbd-mirror 2024-03-26T21:09:06.892 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:06 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:06.892 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:06 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:09:06.892 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:06 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:09:06.892 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:06 smithi063 ceph-mon[66934]: Upgrade: Setting container_image for all ceph-exporter 2024-03-26T21:09:06.892 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:06 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:06.892 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:06 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:09:06.892 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:06 smithi063 ceph-mon[66934]: Upgrade: Setting container_image for all iscsi 2024-03-26T21:09:06.892 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:06 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:07.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:06 smithi174 ceph-mon[53014]: Detected new or changed devices on smithi174 2024-03-26T21:09:07.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:06 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:07.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:06 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:07.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:06 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:09:07.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:06 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T21:09:07.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:06 smithi174 ceph-mon[53014]: pgmap v330: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.8 KiB/s rd, 7.7 KiB/s wr, 6 op/s 2024-03-26T21:09:07.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:06 smithi174 ceph-mon[53014]: pgmap v331: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 209 B/s rd, 1.7 KiB/s wr, 0 op/s 2024-03-26T21:09:07.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:06 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:07.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:06 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:07.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:06 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:09:07.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:06 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:09:07.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:06 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:09:07.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:06 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:09:07.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:06 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:09:07.050 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:06 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:09:07.050 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:06 smithi174 ceph-mon[53014]: Upgrade: Setting container_image for all mds 2024-03-26T21:09:07.050 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:06 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:07.050 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:06 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi063.tzaznz"}]: dispatch 2024-03-26T21:09:07.050 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:06 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi063.tzaznz"}]': finished 2024-03-26T21:09:07.050 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:06 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi174.gitdvk"}]: dispatch 2024-03-26T21:09:07.050 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:06 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi174.gitdvk"}]': finished 2024-03-26T21:09:07.050 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:06 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:09:07.050 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:06 smithi174 ceph-mon[53014]: Upgrade: Setting container_image for all rgw 2024-03-26T21:09:07.050 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:06 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:07.050 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:06 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:09:07.050 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:06 smithi174 ceph-mon[53014]: Upgrade: Setting container_image for all rbd-mirror 2024-03-26T21:09:07.050 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:06 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:07.051 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:06 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:09:07.051 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:06 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:09:07.051 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:06 smithi174 ceph-mon[53014]: Upgrade: Setting container_image for all ceph-exporter 2024-03-26T21:09:07.051 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:06 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:07.051 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:06 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:09:07.051 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:06 smithi174 ceph-mon[53014]: Upgrade: Setting container_image for all iscsi 2024-03-26T21:09:07.051 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:06 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:09.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:08 smithi174 ceph-mon[53014]: pgmap v332: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 209 B/s rd, 1.7 KiB/s wr, 0 op/s 2024-03-26T21:09:09.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:08 smithi174 ceph-mon[53014]: Upgrade: Updating nfs.foo.0.0.smithi063.fgmpwa (1/2) 2024-03-26T21:09:09.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:08 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:09.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:08 smithi174 ceph-mon[53014]: Creating key for client.nfs.foo.0.0.smithi063.fgmpwa 2024-03-26T21:09:09.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:08 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi063.fgmpwa", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-03-26T21:09:09.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:08 smithi174 ceph-mon[53014]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-03-26T21:09:09.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:08 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-03-26T21:09:09.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:08 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-03-26T21:09:09.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:08 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:09:09.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:08 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-03-26T21:09:09.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:08 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-03-26T21:09:09.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:08 smithi174 ceph-mon[53014]: Rados config object exists: conf-nfs.foo 2024-03-26T21:09:09.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:08 smithi174 ceph-mon[53014]: Creating key for client.nfs.foo.0.0.smithi063.fgmpwa-rgw 2024-03-26T21:09:09.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:08 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi063.fgmpwa-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-03-26T21:09:09.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:08 smithi174 ceph-mon[53014]: Bind address in nfs.foo.0.0.smithi063.fgmpwa's ganesha conf is defaulting to empty 2024-03-26T21:09:09.300 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:08 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:09:09.300 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:08 smithi174 ceph-mon[53014]: Deploying daemon nfs.foo.0.0.smithi063.fgmpwa on smithi063 2024-03-26T21:09:09.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:08 smithi063 ceph-mon[66934]: pgmap v332: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 209 B/s rd, 1.7 KiB/s wr, 0 op/s 2024-03-26T21:09:09.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:08 smithi063 ceph-mon[66934]: Upgrade: Updating nfs.foo.0.0.smithi063.fgmpwa (1/2) 2024-03-26T21:09:09.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:08 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:09.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:08 smithi063 ceph-mon[66934]: Creating key for client.nfs.foo.0.0.smithi063.fgmpwa 2024-03-26T21:09:09.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:08 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi063.fgmpwa", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-03-26T21:09:09.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:08 smithi063 ceph-mon[66934]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-03-26T21:09:09.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:08 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-03-26T21:09:09.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:08 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-03-26T21:09:09.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:08 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:09:09.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:08 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-03-26T21:09:09.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:08 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-03-26T21:09:09.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:08 smithi063 ceph-mon[66934]: Rados config object exists: conf-nfs.foo 2024-03-26T21:09:09.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:08 smithi063 ceph-mon[66934]: Creating key for client.nfs.foo.0.0.smithi063.fgmpwa-rgw 2024-03-26T21:09:09.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:08 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi063.fgmpwa-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-03-26T21:09:09.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:08 smithi063 ceph-mon[66934]: Bind address in nfs.foo.0.0.smithi063.fgmpwa's ganesha conf is defaulting to empty 2024-03-26T21:09:09.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:08 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:09:09.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:08 smithi063 ceph-mon[66934]: Deploying daemon nfs.foo.0.0.smithi063.fgmpwa on smithi063 2024-03-26T21:09:10.942 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:10 smithi063 ceph-mon[66934]: pgmap v333: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 209 B/s rd, 209 B/s wr, 0 op/s 2024-03-26T21:09:10.942 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:10 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:10.942 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:10 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:09:11.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:10 smithi174 ceph-mon[53014]: pgmap v333: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 209 B/s rd, 209 B/s wr, 0 op/s 2024-03-26T21:09:11.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:10 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:11.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:10 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:09:13.203 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:12 smithi174 ceph-mon[53014]: pgmap v334: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1.3 KiB/s rd, 524 B/s wr, 1 op/s 2024-03-26T21:09:13.204 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:12 smithi174 ceph-mon[53014]: Evicting (and blocklisting) client session 14530 (172.21.15.63:0/4061700709) 2024-03-26T21:09:13.204 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:12 smithi174 ceph-mon[53014]: from='mds.? [v2:172.21.15.63:6834/3891894564,v1:172.21.15.63:6835/3891894564]' entity='mds.foofs.smithi063.tzaznz' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.63:0/4061700709"}]: dispatch 2024-03-26T21:09:13.204 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:12 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:13.204 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:12 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:13.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:12 smithi063 ceph-mon[66934]: pgmap v334: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1.3 KiB/s rd, 524 B/s wr, 1 op/s 2024-03-26T21:09:13.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:12 smithi063 ceph-mon[66934]: Evicting (and blocklisting) client session 14530 (172.21.15.63:0/4061700709) 2024-03-26T21:09:13.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:12 smithi063 ceph-mon[66934]: from='mds.? [v2:172.21.15.63:6834/3891894564,v1:172.21.15.63:6835/3891894564]' entity='mds.foofs.smithi063.tzaznz' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.63:0/4061700709"}]: dispatch 2024-03-26T21:09:13.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:12 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:13.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:12 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:14.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:13 smithi174 ceph-mon[53014]: from='mds.? [v2:172.21.15.63:6834/3891894564,v1:172.21.15.63:6835/3891894564]' entity='mds.foofs.smithi063.tzaznz' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.63:0/4061700709"}]': finished 2024-03-26T21:09:14.302 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:13 smithi174 ceph-mon[53014]: osdmap e156: 8 total, 8 up, 8 in 2024-03-26T21:09:14.303 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:13 smithi063 ceph-mon[66934]: from='mds.? [v2:172.21.15.63:6834/3891894564,v1:172.21.15.63:6835/3891894564]' entity='mds.foofs.smithi063.tzaznz' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.63:0/4061700709"}]': finished 2024-03-26T21:09:14.303 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:13 smithi063 ceph-mon[66934]: osdmap e156: 8 total, 8 up, 8 in 2024-03-26T21:09:15.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:15 smithi174 ceph-mon[53014]: pgmap v336: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1.6 KiB/s rd, 383 B/s wr, 1 op/s 2024-03-26T21:09:15.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:15 smithi174 ceph-mon[53014]: Upgrade: Updating nfs.foo.1.0.smithi174.pytptt (2/2) 2024-03-26T21:09:15.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:15 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:15.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:15 smithi174 ceph-mon[53014]: Creating key for client.nfs.foo.1.0.smithi174.pytptt 2024-03-26T21:09:15.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:15 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi174.pytptt", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-03-26T21:09:15.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:15 smithi174 ceph-mon[53014]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-03-26T21:09:15.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:15 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-03-26T21:09:15.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:15 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-03-26T21:09:15.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:15 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:09:15.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:15 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-03-26T21:09:15.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:15 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-03-26T21:09:15.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:15 smithi174 ceph-mon[53014]: Rados config object exists: conf-nfs.foo 2024-03-26T21:09:15.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:15 smithi174 ceph-mon[53014]: Creating key for client.nfs.foo.1.0.smithi174.pytptt-rgw 2024-03-26T21:09:15.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:15 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi174.pytptt-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-03-26T21:09:15.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:15 smithi174 ceph-mon[53014]: Bind address in nfs.foo.1.0.smithi174.pytptt's ganesha conf is defaulting to empty 2024-03-26T21:09:15.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:15 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:09:15.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:15 smithi174 ceph-mon[53014]: Deploying daemon nfs.foo.1.0.smithi174.pytptt on smithi174 2024-03-26T21:09:15.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:15 smithi063 ceph-mon[66934]: pgmap v336: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1.6 KiB/s rd, 383 B/s wr, 1 op/s 2024-03-26T21:09:15.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:15 smithi063 ceph-mon[66934]: Upgrade: Updating nfs.foo.1.0.smithi174.pytptt (2/2) 2024-03-26T21:09:15.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:15 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:15.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:15 smithi063 ceph-mon[66934]: Creating key for client.nfs.foo.1.0.smithi174.pytptt 2024-03-26T21:09:15.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:15 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi174.pytptt", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-03-26T21:09:15.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:15 smithi063 ceph-mon[66934]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-03-26T21:09:15.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:15 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-03-26T21:09:15.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:15 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-03-26T21:09:15.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:15 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:09:15.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:15 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-03-26T21:09:15.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:15 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-03-26T21:09:15.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:15 smithi063 ceph-mon[66934]: Rados config object exists: conf-nfs.foo 2024-03-26T21:09:15.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:15 smithi063 ceph-mon[66934]: Creating key for client.nfs.foo.1.0.smithi174.pytptt-rgw 2024-03-26T21:09:15.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:15 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi174.pytptt-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-03-26T21:09:15.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:15 smithi063 ceph-mon[66934]: Bind address in nfs.foo.1.0.smithi174.pytptt's ganesha conf is defaulting to empty 2024-03-26T21:09:15.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:15 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:09:15.302 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:15 smithi063 ceph-mon[66934]: Deploying daemon nfs.foo.1.0.smithi174.pytptt on smithi174 2024-03-26T21:09:17.221 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:17 smithi174 ceph-mon[53014]: pgmap v337: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1.5 KiB/s rd, 1023 B/s wr, 2 op/s 2024-03-26T21:09:17.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:17 smithi063 ceph-mon[66934]: pgmap v337: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1.5 KiB/s rd, 1023 B/s wr, 2 op/s 2024-03-26T21:09:17.737 INFO:teuthology.orchestra.run.smithi063.stdout:true 2024-03-26T21:09:18.544 INFO:teuthology.orchestra.run.smithi063.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-26T21:09:18.544 INFO:teuthology.orchestra.run.smithi063.stdout:alertmanager.smithi063 smithi063 *:9093,9094 running (10m) 39s ago 19m 24.7M - 0.23.0 ba2b418f427c 6e160c94aac3 2024-03-26T21:09:18.544 INFO:teuthology.orchestra.run.smithi063.stdout:crash.smithi063 smithi063 running (6m) 39s ago 19m 6665k - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 cf1096d84270 2024-03-26T21:09:18.544 INFO:teuthology.orchestra.run.smithi063.stdout:crash.smithi174 smithi174 running (6m) 17s ago 19m 6665k - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 cb5371a92684 2024-03-26T21:09:18.544 INFO:teuthology.orchestra.run.smithi063.stdout:grafana.smithi063 smithi063 *:3000 error 39s ago 19m - - 2024-03-26T21:09:18.544 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foofs.smithi063.tzaznz smithi063 running (45s) 39s ago 13m 11.4M - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 ef76ec94effe 2024-03-26T21:09:18.544 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foofs.smithi174.gitdvk smithi174 running (20s) 17s ago 13m 13.4M - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 1ac021fe2172 2024-03-26T21:09:18.544 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.smithi063.lxhujt smithi063 *:8443,9283,8765 running (9m) 39s ago 20m 514M - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 6d9d10330a90 2024-03-26T21:09:18.544 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.smithi174.sexoin smithi174 *:8443,9283,8765 running (8m) 17s ago 18m 453M - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 d7546233672d 2024-03-26T21:09:18.545 INFO:teuthology.orchestra.run.smithi063.stdout:mon.smithi063 smithi063 running (8m) 39s ago 21m 62.1M 2048M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 c644200344f8 2024-03-26T21:09:18.545 INFO:teuthology.orchestra.run.smithi063.stdout:mon.smithi174 smithi174 running (7m) 17s ago 18m 49.7M 2048M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 1a6d442692e7 2024-03-26T21:09:18.545 INFO:teuthology.orchestra.run.smithi063.stdout:nfs.foo.0.0.smithi063.fgmpwa smithi063 *:2049 starting - - - - 2024-03-26T21:09:18.545 INFO:teuthology.orchestra.run.smithi063.stdout:nfs.foo.1.0.smithi174.pytptt smithi174 *:2049 starting - - - - 2024-03-26T21:09:18.545 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.smithi063 smithi063 *:9100 running (10m) 39s ago 19m 19.8M - 1.3.1 1dbe0e931976 f52f41fd14c0 2024-03-26T21:09:18.545 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.smithi174 smithi174 *:9100 running (9m) 17s ago 18m 18.8M - 1.3.1 1dbe0e931976 0c027f39a4c8 2024-03-26T21:09:18.545 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0 smithi063 running (5m) 39s ago 18m 416M 1936M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 2d4b3fa825cf 2024-03-26T21:09:18.545 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1 smithi063 running (5m) 39s ago 17m 370M 1936M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 d41fbe507197 2024-03-26T21:09:18.545 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2 smithi063 running (4m) 39s ago 17m 347M 1936M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 f123eeabd3fa 2024-03-26T21:09:18.545 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3 smithi063 running (3m) 39s ago 16m 423M 1936M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 953816eafe87 2024-03-26T21:09:18.545 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4 smithi174 running (3m) 17s ago 16m 360M 2704M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 8aa35bff6cea 2024-03-26T21:09:18.545 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5 smithi174 running (2m) 17s ago 15m 313M 2704M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 25dde01e0676 2024-03-26T21:09:18.545 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6 smithi174 running (109s) 17s ago 15m 302M 2704M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 95fa1b77701b 2024-03-26T21:09:18.545 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7 smithi174 running (70s) 17s ago 15m 294M 2704M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 cb89961bc930 2024-03-26T21:09:18.546 INFO:teuthology.orchestra.run.smithi063.stdout:prometheus.smithi063 smithi063 *:9095 running (8m) 39s ago 18m 76.2M - 2.33.4 514e6a882f6e 567eef202627 2024-03-26T21:09:19.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:18 smithi174 ceph-mon[53014]: from='client.34530 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:09:19.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:18 smithi174 ceph-mon[53014]: pgmap v338: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3.3 KiB/s rd, 1.4 KiB/s wr, 5 op/s 2024-03-26T21:09:19.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:18 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:19.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:18 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:19.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:18 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:09:19.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:18 smithi174 ceph-mon[53014]: from='client.34542 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:09:19.160 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-03-26T21:09:19.160 INFO:teuthology.orchestra.run.smithi063.stdout: "mon": { 2024-03-26T21:09:19.160 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 2 2024-03-26T21:09:19.160 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:09:19.160 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr": { 2024-03-26T21:09:19.160 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 2 2024-03-26T21:09:19.160 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:09:19.161 INFO:teuthology.orchestra.run.smithi063.stdout: "osd": { 2024-03-26T21:09:19.161 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 8 2024-03-26T21:09:19.161 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:09:19.161 INFO:teuthology.orchestra.run.smithi063.stdout: "mds": { 2024-03-26T21:09:19.161 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 2 2024-03-26T21:09:19.161 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:09:19.161 INFO:teuthology.orchestra.run.smithi063.stdout: "overall": { 2024-03-26T21:09:19.161 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 14 2024-03-26T21:09:19.161 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-03-26T21:09:19.161 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-03-26T21:09:19.161 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:18 smithi063 ceph-mon[66934]: from='client.34530 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:09:19.161 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:18 smithi063 ceph-mon[66934]: pgmap v338: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3.3 KiB/s rd, 1.4 KiB/s wr, 5 op/s 2024-03-26T21:09:19.162 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:18 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:19.162 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:18 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:19.162 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:18 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:09:19.162 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:18 smithi063 ceph-mon[66934]: from='client.34542 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:09:19.556 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-03-26T21:09:19.557 INFO:teuthology.orchestra.run.smithi063.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d", 2024-03-26T21:09:19.557 INFO:teuthology.orchestra.run.smithi063.stdout: "in_progress": true, 2024-03-26T21:09:19.557 INFO:teuthology.orchestra.run.smithi063.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-26T21:09:19.557 INFO:teuthology.orchestra.run.smithi063.stdout: "services_complete": [ 2024-03-26T21:09:19.557 INFO:teuthology.orchestra.run.smithi063.stdout: "mon", 2024-03-26T21:09:19.557 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr", 2024-03-26T21:09:19.557 INFO:teuthology.orchestra.run.smithi063.stdout: "osd", 2024-03-26T21:09:19.557 INFO:teuthology.orchestra.run.smithi063.stdout: "crash", 2024-03-26T21:09:19.557 INFO:teuthology.orchestra.run.smithi063.stdout: "mds" 2024-03-26T21:09:19.557 INFO:teuthology.orchestra.run.smithi063.stdout: ], 2024-03-26T21:09:19.557 INFO:teuthology.orchestra.run.smithi063.stdout: "progress": "16/23 daemons upgraded", 2024-03-26T21:09:19.557 INFO:teuthology.orchestra.run.smithi063.stdout: "message": "Currently upgrading nfs daemons", 2024-03-26T21:09:19.557 INFO:teuthology.orchestra.run.smithi063.stdout: "is_paused": false 2024-03-26T21:09:19.557 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-03-26T21:09:20.062 INFO:teuthology.orchestra.run.smithi063.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-26T21:09:20.062 INFO:teuthology.orchestra.run.smithi063.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-26T21:09:20.062 INFO:teuthology.orchestra.run.smithi063.stdout: daemon grafana.smithi063 on smithi063 is in error state 2024-03-26T21:09:20.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:19 smithi174 ceph-mon[53014]: from='client.34546 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:09:20.313 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:19 smithi174 ceph-mon[53014]: from='client.? 172.21.15.63:0/198258441' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:09:20.314 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:19 smithi063 ceph-mon[66934]: from='client.34546 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:09:20.314 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:19 smithi063 ceph-mon[66934]: from='client.? 172.21.15.63:0/198258441' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:09:21.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:20 smithi174 ceph-mon[53014]: from='client.34554 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:09:21.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:20 smithi174 ceph-mon[53014]: pgmap v339: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3.3 KiB/s rd, 1.4 KiB/s wr, 5 op/s 2024-03-26T21:09:21.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:20 smithi174 ceph-mon[53014]: from='client.? 172.21.15.63:0/2490956322' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-26T21:09:21.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:20 smithi174 ceph-mon[53014]: from='mds.? [v2:172.21.15.63:6834/3891894564,v1:172.21.15.63:6835/3891894564]' entity='mds.foofs.smithi063.tzaznz' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.174:0/4163182094"}]: dispatch 2024-03-26T21:09:21.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:20 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:21.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:20 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:21.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:20 smithi063 ceph-mon[66934]: from='client.34554 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:09:21.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:20 smithi063 ceph-mon[66934]: pgmap v339: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3.3 KiB/s rd, 1.4 KiB/s wr, 5 op/s 2024-03-26T21:09:21.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:20 smithi063 ceph-mon[66934]: from='client.? 172.21.15.63:0/2490956322' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-26T21:09:21.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:20 smithi063 ceph-mon[66934]: from='mds.? [v2:172.21.15.63:6834/3891894564,v1:172.21.15.63:6835/3891894564]' entity='mds.foofs.smithi063.tzaznz' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.174:0/4163182094"}]: dispatch 2024-03-26T21:09:21.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:20 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:21.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:20 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:22.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:21 smithi063 ceph-mon[66934]: Evicting (and blocklisting) client session 24365 (172.21.15.174:0/4163182094) 2024-03-26T21:09:22.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:21 smithi063 ceph-mon[66934]: from='mds.? [v2:172.21.15.63:6834/3891894564,v1:172.21.15.63:6835/3891894564]' entity='mds.foofs.smithi063.tzaznz' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.174:0/4163182094"}]': finished 2024-03-26T21:09:22.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:21 smithi063 ceph-mon[66934]: osdmap e157: 8 total, 8 up, 8 in 2024-03-26T21:09:22.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:21 smithi174 ceph-mon[53014]: Evicting (and blocklisting) client session 24365 (172.21.15.174:0/4163182094) 2024-03-26T21:09:22.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:21 smithi174 ceph-mon[53014]: from='mds.? [v2:172.21.15.63:6834/3891894564,v1:172.21.15.63:6835/3891894564]' entity='mds.foofs.smithi063.tzaznz' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.174:0/4163182094"}]': finished 2024-03-26T21:09:22.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:21 smithi174 ceph-mon[53014]: osdmap e157: 8 total, 8 up, 8 in 2024-03-26T21:09:23.107 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:22 smithi174 ceph-mon[53014]: pgmap v341: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4.8 KiB/s rd, 1.9 KiB/s wr, 7 op/s 2024-03-26T21:09:23.108 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:22 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:23.108 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:22 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:23.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:22 smithi063 ceph-mon[66934]: pgmap v341: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4.8 KiB/s rd, 1.9 KiB/s wr, 7 op/s 2024-03-26T21:09:23.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:22 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:23.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:22 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:24.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:24 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:24.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:24 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:24.551 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:24 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:24.551 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:24 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:25.243 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:25 smithi063 ceph-mon[66934]: pgmap v342: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4.3 KiB/s rd, 1.7 KiB/s wr, 6 op/s 2024-03-26T21:09:25.244 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:25 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:25.244 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:25 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:25.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:25 smithi174 ceph-mon[53014]: pgmap v342: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4.3 KiB/s rd, 1.7 KiB/s wr, 6 op/s 2024-03-26T21:09:25.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:25 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:25.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:25 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:26.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:26 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:26.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:26 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:26.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:26 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:09:26.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:26 smithi174 ceph-mon[53014]: pgmap v343: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4.1 KiB/s rd, 1.4 KiB/s wr, 6 op/s 2024-03-26T21:09:26.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:26 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:26.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:26 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:26.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:26 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:09:26.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:26 smithi063 ceph-mon[66934]: pgmap v343: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4.1 KiB/s rd, 1.4 KiB/s wr, 6 op/s 2024-03-26T21:09:29.268 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:29 smithi063 ceph-mon[66934]: pgmap v344: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.3 KiB/s rd, 1023 B/s wr, 3 op/s 2024-03-26T21:09:29.268 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:29 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:29.268 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:29 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:29.268 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:29 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:09:29.268 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:29 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T21:09:29.268 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:29 smithi063 ceph-mon[66934]: pgmap v345: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.7 KiB/s rd, 1.2 KiB/s wr, 4 op/s 2024-03-26T21:09:29.269 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:29 smithi063 ceph-mon[66934]: pgmap v346: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3.0 KiB/s rd, 1.4 KiB/s wr, 4 op/s 2024-03-26T21:09:29.269 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:29 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:29.269 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:29 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:29.269 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:29 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:09:29.269 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:29 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:09:29.269 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:29 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:09:29.269 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:29 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:09:29.269 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:29 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:09:29.269 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:29 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:09:29.269 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:29 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:09:29.269 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:29 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:09:29.269 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:29 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:09:29.269 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:29 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:09:29.269 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:29 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:09:29.269 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:29 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:09:29.270 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:29 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:29.270 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:29 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi063.fgmpwa"}]: dispatch 2024-03-26T21:09:29.270 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:29 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi063.fgmpwa"}]': finished 2024-03-26T21:09:29.270 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:29 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi174.pytptt"}]: dispatch 2024-03-26T21:09:29.270 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:29 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi174.pytptt"}]': finished 2024-03-26T21:09:29.270 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:29 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:09:29.270 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:29 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:29.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:29 smithi174 ceph-mon[53014]: pgmap v344: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.3 KiB/s rd, 1023 B/s wr, 3 op/s 2024-03-26T21:09:29.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:29 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:29.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:29 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:29.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:29 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:09:29.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:29 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T21:09:29.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:29 smithi174 ceph-mon[53014]: pgmap v345: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.7 KiB/s rd, 1.2 KiB/s wr, 4 op/s 2024-03-26T21:09:29.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:29 smithi174 ceph-mon[53014]: pgmap v346: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3.0 KiB/s rd, 1.4 KiB/s wr, 4 op/s 2024-03-26T21:09:29.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:29 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:29.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:29 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:29.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:29 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:09:29.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:29 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:09:29.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:29 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:09:29.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:29 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:09:29.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:29 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:09:29.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:29 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:09:29.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:29 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:09:29.550 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:29 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:09:29.550 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:29 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:09:29.550 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:29 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:09:29.550 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:29 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:09:29.550 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:29 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:09:29.550 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:29 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:29.550 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:29 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi063.fgmpwa"}]: dispatch 2024-03-26T21:09:29.550 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:29 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi063.fgmpwa"}]': finished 2024-03-26T21:09:29.550 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:29 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi174.pytptt"}]: dispatch 2024-03-26T21:09:29.550 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:29 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi174.pytptt"}]': finished 2024-03-26T21:09:29.550 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:29 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:09:29.550 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:29 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:30.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:30 smithi063 ceph-mon[66934]: Upgrade: Setting container_image for all nfs 2024-03-26T21:09:30.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:30 smithi063 ceph-mon[66934]: Upgrade: Setting container_image for all nvmeof 2024-03-26T21:09:30.449 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:30 smithi174 ceph-mon[53014]: Upgrade: Setting container_image for all nfs 2024-03-26T21:09:30.450 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:30 smithi174 ceph-mon[53014]: Upgrade: Setting container_image for all nvmeof 2024-03-26T21:09:31.394 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:31 smithi063 ceph-mon[66934]: Upgrade: Updating node-exporter.smithi063 (1/2) 2024-03-26T21:09:31.394 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:31 smithi063 ceph-mon[66934]: Deploying daemon node-exporter.smithi063 on smithi063 2024-03-26T21:09:31.395 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:31 smithi063 ceph-mon[66934]: pgmap v347: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 487 B/s wr, 0 op/s 2024-03-26T21:09:31.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:31 smithi174 ceph-mon[53014]: Upgrade: Updating node-exporter.smithi063 (1/2) 2024-03-26T21:09:31.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:31 smithi174 ceph-mon[53014]: Deploying daemon node-exporter.smithi063 on smithi063 2024-03-26T21:09:31.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:31 smithi174 ceph-mon[53014]: pgmap v347: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 487 B/s wr, 0 op/s 2024-03-26T21:09:33.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:33 smithi174 ceph-mon[53014]: pgmap v348: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 243 B/s rd, 487 B/s wr, 0 op/s 2024-03-26T21:09:33.551 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:33 smithi063 ceph-mon[66934]: pgmap v348: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 243 B/s rd, 487 B/s wr, 0 op/s 2024-03-26T21:09:35.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:35 smithi063 ceph-mon[66934]: pgmap v349: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 243 B/s rd, 0 op/s 2024-03-26T21:09:35.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:35 smithi174 ceph-mon[53014]: pgmap v349: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 243 B/s rd, 0 op/s 2024-03-26T21:09:36.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:36 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:36.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:36 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:36.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:36 smithi174 ceph-mon[53014]: pgmap v350: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 243 B/s rd, 0 op/s 2024-03-26T21:09:36.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:36 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:36.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:36 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:36.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:36 smithi063 ceph-mon[66934]: pgmap v350: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 243 B/s rd, 0 op/s 2024-03-26T21:09:37.743 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:37 smithi174 ceph-mon[53014]: Upgrade: Updating node-exporter.smithi174 (2/2) 2024-03-26T21:09:37.743 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:37 smithi174 ceph-mon[53014]: Deploying daemon node-exporter.smithi174 on smithi174 2024-03-26T21:09:37.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:37 smithi063 ceph-mon[66934]: Upgrade: Updating node-exporter.smithi174 (2/2) 2024-03-26T21:09:37.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:37 smithi063 ceph-mon[66934]: Deploying daemon node-exporter.smithi174 on smithi174 2024-03-26T21:09:38.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:38 smithi174 ceph-mon[53014]: pgmap v351: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-03-26T21:09:38.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:38 smithi063 ceph-mon[66934]: pgmap v351: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-03-26T21:09:41.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:40 smithi174 ceph-mon[53014]: pgmap v352: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:09:41.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:40 smithi063 ceph-mon[66934]: pgmap v352: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:09:41.973 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:41 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:09:42.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:41 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:09:43.226 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:43 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:43.226 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:43 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:43.226 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:43 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:09:43.226 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:43 smithi174 ceph-mon[53014]: pgmap v353: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-26T21:09:43.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:43 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:43.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:43 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:43.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:43 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:09:43.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:43 smithi063 ceph-mon[66934]: pgmap v353: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-26T21:09:45.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:45 smithi174 ceph-mon[53014]: pgmap v354: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:09:45.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:45 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:45.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:45 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:45.551 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:45 smithi063 ceph-mon[66934]: pgmap v354: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:09:45.551 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:45 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:45.551 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:45 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:47.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:47 smithi174 ceph-mon[53014]: pgmap v355: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:09:47.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:47 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:47.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:47 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:47.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:47 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:47.549 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:47 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:47.551 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:47 smithi063 ceph-mon[66934]: pgmap v355: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:09:47.551 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:47 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:47.551 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:47 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:47.551 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:47 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:47.551 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:47 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:49.472 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:49 smithi174 ceph-mon[53014]: pgmap v356: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:09:49.473 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:49 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:49.473 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:49 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:49.551 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:49 smithi063 ceph-mon[66934]: pgmap v356: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:09:49.551 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:49 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:49.551 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:49 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:50.451 INFO:teuthology.orchestra.run.smithi063.stdout:true 2024-03-26T21:09:50.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:50 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:50.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:50 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:50.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:50 smithi174 ceph-mon[53014]: pgmap v357: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:09:50.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:50 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:50.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:50 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:50.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:50 smithi063 ceph-mon[66934]: pgmap v357: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:09:51.339 INFO:teuthology.orchestra.run.smithi063.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-26T21:09:51.340 INFO:teuthology.orchestra.run.smithi063.stdout:alertmanager.smithi063 smithi063 *:9093,9094 running (11m) 4s ago 20m 24.7M - 0.23.0 ba2b418f427c 6e160c94aac3 2024-03-26T21:09:51.340 INFO:teuthology.orchestra.run.smithi063.stdout:crash.smithi063 smithi063 running (7m) 4s ago 20m 6665k - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 cf1096d84270 2024-03-26T21:09:51.340 INFO:teuthology.orchestra.run.smithi063.stdout:crash.smithi174 smithi174 running (7m) 6s ago 19m 6665k - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 cb5371a92684 2024-03-26T21:09:51.340 INFO:teuthology.orchestra.run.smithi063.stdout:grafana.smithi063 smithi063 *:3000 error 4s ago 19m - - 2024-03-26T21:09:51.340 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foofs.smithi063.tzaznz smithi063 running (78s) 4s ago 14m 32.0M - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 ef76ec94effe 2024-03-26T21:09:51.340 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foofs.smithi174.gitdvk smithi174 running (53s) 6s ago 14m 14.7M - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 1ac021fe2172 2024-03-26T21:09:51.340 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.smithi063.lxhujt smithi063 *:8443,9283,8765 running (10m) 4s ago 21m 516M - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 6d9d10330a90 2024-03-26T21:09:51.340 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.smithi174.sexoin smithi174 *:8443,9283,8765 running (9m) 6s ago 19m 453M - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 d7546233672d 2024-03-26T21:09:51.340 INFO:teuthology.orchestra.run.smithi063.stdout:mon.smithi063 smithi063 running (8m) 4s ago 21m 67.2M 2048M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 c644200344f8 2024-03-26T21:09:51.340 INFO:teuthology.orchestra.run.smithi063.stdout:mon.smithi174 smithi174 running (8m) 6s ago 19m 53.5M 2048M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 1a6d442692e7 2024-03-26T21:09:51.340 INFO:teuthology.orchestra.run.smithi063.stdout:nfs.foo.0.0.smithi063.fgmpwa smithi063 *:2049 running (39s) 4s ago 13m 69.0M - 5.7 a9bdc3eaeb10 5944f2d65c1f 2024-03-26T21:09:51.340 INFO:teuthology.orchestra.run.smithi063.stdout:nfs.foo.1.0.smithi174.pytptt smithi174 *:2049 running (33s) 6s ago 13m 68.5M - 5.7 a9bdc3eaeb10 e604d7340c73 2024-03-26T21:09:51.340 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.smithi063 smithi063 *:9100 running (16s) 4s ago 19m 7998k - 1.5.0 0da6a335fe13 fead24e0db0a 2024-03-26T21:09:51.341 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.smithi174 smithi174 *:9100 running (9s) 6s ago 19m 3586k - 1.5.0 0da6a335fe13 91e70091b852 2024-03-26T21:09:51.341 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0 smithi063 running (6m) 4s ago 18m 419M 1936M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 2d4b3fa825cf 2024-03-26T21:09:51.341 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1 smithi063 running (5m) 4s ago 18m 373M 1936M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 d41fbe507197 2024-03-26T21:09:51.341 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2 smithi063 running (4m) 4s ago 17m 348M 1936M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 f123eeabd3fa 2024-03-26T21:09:51.341 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3 smithi063 running (4m) 4s ago 17m 426M 1936M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 953816eafe87 2024-03-26T21:09:51.341 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4 smithi174 running (3m) 6s ago 16m 360M 2704M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 8aa35bff6cea 2024-03-26T21:09:51.341 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5 smithi174 running (2m) 6s ago 16m 312M 2704M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 25dde01e0676 2024-03-26T21:09:51.341 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6 smithi174 running (2m) 6s ago 16m 301M 2704M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 95fa1b77701b 2024-03-26T21:09:51.341 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7 smithi174 running (102s) 6s ago 15m 294M 2704M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 cb89961bc930 2024-03-26T21:09:51.341 INFO:teuthology.orchestra.run.smithi063.stdout:prometheus.smithi063 smithi063 *:9095 running (9m) 4s ago 19m 78.3M - 2.33.4 514e6a882f6e 567eef202627 2024-03-26T21:09:51.796 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-03-26T21:09:51.796 INFO:teuthology.orchestra.run.smithi063.stdout: "mon": { 2024-03-26T21:09:51.796 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 2 2024-03-26T21:09:51.796 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:09:51.796 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr": { 2024-03-26T21:09:51.796 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 2 2024-03-26T21:09:51.796 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:09:51.796 INFO:teuthology.orchestra.run.smithi063.stdout: "osd": { 2024-03-26T21:09:51.796 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 8 2024-03-26T21:09:51.796 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:09:51.796 INFO:teuthology.orchestra.run.smithi063.stdout: "mds": { 2024-03-26T21:09:51.797 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 2 2024-03-26T21:09:51.797 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:09:51.797 INFO:teuthology.orchestra.run.smithi063.stdout: "overall": { 2024-03-26T21:09:51.797 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 14 2024-03-26T21:09:51.797 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-03-26T21:09:51.797 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-03-26T21:09:51.797 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:51 smithi063 ceph-mon[66934]: from='client.34566 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:09:51.797 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:51 smithi063 ceph-mon[66934]: from='client.34570 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:09:51.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:51 smithi174 ceph-mon[53014]: from='client.34566 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:09:51.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:51 smithi174 ceph-mon[53014]: from='client.34570 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:09:52.187 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-03-26T21:09:52.187 INFO:teuthology.orchestra.run.smithi063.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d", 2024-03-26T21:09:52.187 INFO:teuthology.orchestra.run.smithi063.stdout: "in_progress": true, 2024-03-26T21:09:52.187 INFO:teuthology.orchestra.run.smithi063.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-26T21:09:52.187 INFO:teuthology.orchestra.run.smithi063.stdout: "services_complete": [ 2024-03-26T21:09:52.188 INFO:teuthology.orchestra.run.smithi063.stdout: "nfs", 2024-03-26T21:09:52.188 INFO:teuthology.orchestra.run.smithi063.stdout: "mon", 2024-03-26T21:09:52.188 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr", 2024-03-26T21:09:52.188 INFO:teuthology.orchestra.run.smithi063.stdout: "osd", 2024-03-26T21:09:52.188 INFO:teuthology.orchestra.run.smithi063.stdout: "crash", 2024-03-26T21:09:52.188 INFO:teuthology.orchestra.run.smithi063.stdout: "mds" 2024-03-26T21:09:52.188 INFO:teuthology.orchestra.run.smithi063.stdout: ], 2024-03-26T21:09:52.188 INFO:teuthology.orchestra.run.smithi063.stdout: "progress": "18/23 daemons upgraded", 2024-03-26T21:09:52.188 INFO:teuthology.orchestra.run.smithi063.stdout: "message": "Currently upgrading node-exporter daemons", 2024-03-26T21:09:52.188 INFO:teuthology.orchestra.run.smithi063.stdout: "is_paused": false 2024-03-26T21:09:52.188 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-03-26T21:09:52.630 INFO:teuthology.orchestra.run.smithi063.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-26T21:09:52.634 INFO:teuthology.orchestra.run.smithi063.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-26T21:09:52.634 INFO:teuthology.orchestra.run.smithi063.stdout: daemon grafana.smithi063 on smithi063 is in error state 2024-03-26T21:09:52.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:52 smithi174 ceph-mon[53014]: from='client.34574 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:09:52.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:52 smithi174 ceph-mon[53014]: from='client.? 172.21.15.63:0/2520801814' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:09:52.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:52 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:52.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:52 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:52.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:52 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:09:52.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:52 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T21:09:52.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:52 smithi174 ceph-mon[53014]: pgmap v358: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 344 B/s rd, 0 op/s 2024-03-26T21:09:52.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:52 smithi174 ceph-mon[53014]: pgmap v359: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 207 B/s rd, 0 op/s 2024-03-26T21:09:52.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:52 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:52.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:52 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:52.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:52 smithi174 ceph-mon[53014]: from='client.34582 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:09:52.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:52 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:09:52.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:52 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:09:52.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:52 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:09:52.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:52 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:09:52.800 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:52 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:09:52.800 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:52 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:09:52.800 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:52 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:09:52.800 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:52 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:09:52.800 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:52 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:09:52.800 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:52 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:09:52.800 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:52 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:09:52.800 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:52 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:09:52.800 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:52 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:09:52.800 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:52 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:09:52.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:52 smithi063 ceph-mon[66934]: from='client.34574 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:09:52.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:52 smithi063 ceph-mon[66934]: from='client.? 172.21.15.63:0/2520801814' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:09:52.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:52 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:52.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:52 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:52.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:52 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:09:52.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:52 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T21:09:52.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:52 smithi063 ceph-mon[66934]: pgmap v358: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 344 B/s rd, 0 op/s 2024-03-26T21:09:52.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:52 smithi063 ceph-mon[66934]: pgmap v359: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 207 B/s rd, 0 op/s 2024-03-26T21:09:52.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:52 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:52.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:52 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:09:52.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:52 smithi063 ceph-mon[66934]: from='client.34582 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:09:52.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:52 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:09:52.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:52 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:09:52.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:52 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:09:52.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:52 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:09:52.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:52 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:09:52.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:52 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:09:52.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:52 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:09:52.802 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:52 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:09:52.803 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:52 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:09:52.803 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:52 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:09:52.803 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:52 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:09:52.803 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:52 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:09:52.803 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:52 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:09:52.803 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:52 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:09:53.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:53 smithi174 ceph-mon[53014]: from='client.? 172.21.15.63:0/3854561819' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-26T21:09:53.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:53 smithi063 ceph-mon[66934]: from='client.? 172.21.15.63:0/3854561819' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-26T21:09:54.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:54 smithi174 ceph-mon[53014]: Upgrade: Updating prometheus.smithi063 2024-03-26T21:09:54.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:54 smithi174 ceph-mon[53014]: pgmap v360: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 207 B/s rd, 0 op/s 2024-03-26T21:09:54.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:54 smithi174 ceph-mon[53014]: Deploying daemon prometheus.smithi063 on smithi063 2024-03-26T21:09:54.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:54 smithi063 ceph-mon[66934]: Upgrade: Updating prometheus.smithi063 2024-03-26T21:09:54.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:54 smithi063 ceph-mon[66934]: pgmap v360: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 207 B/s rd, 0 op/s 2024-03-26T21:09:54.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:54 smithi063 ceph-mon[66934]: Deploying daemon prometheus.smithi063 on smithi063 2024-03-26T21:09:55.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:55 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:09:56.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:55 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:09:57.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:56 smithi174 ceph-mon[53014]: pgmap v361: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 207 B/s rd, 0 op/s 2024-03-26T21:09:57.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:56 smithi063 ceph-mon[66934]: pgmap v361: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 207 B/s rd, 0 op/s 2024-03-26T21:09:59.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:09:59 smithi063 ceph-mon[66934]: pgmap v362: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 207 B/s rd, 0 op/s 2024-03-26T21:09:59.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:09:59 smithi174 ceph-mon[53014]: pgmap v362: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 207 B/s rd, 0 op/s 2024-03-26T21:10:01.259 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:00 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:10:01.284 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:00 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:10:01.284 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:00 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:10:01.284 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:00 smithi063 ceph-mon[66934]: overall HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-26T21:10:01.284 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:00 smithi063 ceph-mon[66934]: pgmap v363: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 207 B/s rd, 0 op/s 2024-03-26T21:10:01.284 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:00 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:10:01.284 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:00 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:10:01.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:00 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:10:01.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:00 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:10:01.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:00 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:10:01.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:00 smithi174 ceph-mon[53014]: overall HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-26T21:10:01.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:00 smithi174 ceph-mon[53014]: pgmap v363: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 207 B/s rd, 0 op/s 2024-03-26T21:10:01.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:00 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:10:01.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:00 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:10:03.431 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:03 smithi063 ceph-mon[66934]: pgmap v364: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-03-26T21:10:03.461 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:03 smithi174 ceph-mon[53014]: pgmap v364: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-03-26T21:10:05.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:05 smithi174 ceph-mon[53014]: pgmap v365: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:10:05.551 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:05 smithi063 ceph-mon[66934]: pgmap v365: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:10:06.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:06 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:10:06.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:06 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:10:06.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:06 smithi174 ceph-mon[53014]: pgmap v366: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:10:06.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:06 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:10:06.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:06 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:10:06.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:06 smithi063 ceph-mon[66934]: pgmap v366: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:10:07.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:07 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:10:07.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:07 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:10:07.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:07 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:10:07.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:07 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:10:08.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:08 smithi174 ceph-mon[53014]: pgmap v367: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:10:08.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:08 smithi063 ceph-mon[66934]: pgmap v367: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:10:11.052 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:10 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:10:11.052 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:10 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:10:11.052 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:10 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:10:11.052 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:10 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T21:10:11.052 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:10 smithi063 ceph-mon[66934]: pgmap v368: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 171 B/s rd, 0 op/s 2024-03-26T21:10:11.052 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:10 smithi063 ceph-mon[66934]: pgmap v369: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 206 B/s rd, 0 op/s 2024-03-26T21:10:11.052 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:10 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:10:11.052 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:10 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:10:11.052 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:10 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-26T21:10:11.052 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:10 smithi063 ceph-mon[66934]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-26T21:10:11.053 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:10 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:10:11.053 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:10 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:11.053 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:10 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:11.053 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:10 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:11.053 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:10 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:11.053 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:10 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:11.053 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:10 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:11.053 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:10 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:11.053 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:10 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:11.053 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:10 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:11.053 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:10 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:11.053 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:10 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:11.053 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:10 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:11.054 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:10 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:11.054 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:10 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:11.054 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:10 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:10:11.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:10 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:10:11.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:10 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:10:11.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:10 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:10:11.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:10 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T21:10:11.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:10 smithi174 ceph-mon[53014]: pgmap v368: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 171 B/s rd, 0 op/s 2024-03-26T21:10:11.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:10 smithi174 ceph-mon[53014]: pgmap v369: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 206 B/s rd, 0 op/s 2024-03-26T21:10:11.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:10 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:10:11.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:10 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:10:11.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:10 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-26T21:10:11.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:10 smithi174 ceph-mon[53014]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-26T21:10:11.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:10 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:10:11.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:10 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:11.299 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:10 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:11.300 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:10 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:11.300 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:10 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:11.300 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:10 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:11.300 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:10 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:11.300 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:10 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:11.300 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:10 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:11.300 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:10 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:11.300 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:10 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:11.300 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:10 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:11.300 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:10 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:11.300 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:10 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:11.300 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:10 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:11.300 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:10 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:10:13.269 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:13 smithi063 ceph-mon[66934]: Upgrade: Updating alertmanager.smithi063 2024-03-26T21:10:13.269 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:13 smithi063 ceph-mon[66934]: Deploying daemon alertmanager.smithi063 on smithi063 2024-03-26T21:10:13.269 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:13 smithi063 ceph-mon[66934]: pgmap v370: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 206 B/s rd, 0 op/s 2024-03-26T21:10:13.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:13 smithi174 ceph-mon[53014]: Upgrade: Updating alertmanager.smithi063 2024-03-26T21:10:13.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:13 smithi174 ceph-mon[53014]: Deploying daemon alertmanager.smithi063 on smithi063 2024-03-26T21:10:13.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:13 smithi174 ceph-mon[53014]: pgmap v370: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 206 B/s rd, 0 op/s 2024-03-26T21:10:15.270 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:15 smithi063 ceph-mon[66934]: pgmap v371: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 206 B/s rd, 0 op/s 2024-03-26T21:10:15.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:15 smithi174 ceph-mon[53014]: pgmap v371: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 206 B/s rd, 0 op/s 2024-03-26T21:10:17.140 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:17 smithi063 ceph-mon[66934]: pgmap v372: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 206 B/s rd, 0 op/s 2024-03-26T21:10:17.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:17 smithi174 ceph-mon[53014]: pgmap v372: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 206 B/s rd, 0 op/s 2024-03-26T21:10:19.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:18 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:10:19.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:18 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:10:19.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:18 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:10:19.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:18 smithi174 ceph-mon[53014]: pgmap v373: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 206 B/s rd, 0 op/s 2024-03-26T21:10:19.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:18 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:10:19.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:18 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:10:19.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:18 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:10:19.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:18 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:10:19.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:18 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:10:19.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:18 smithi063 ceph-mon[66934]: pgmap v373: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 206 B/s rd, 0 op/s 2024-03-26T21:10:19.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:18 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:10:19.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:18 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:10:21.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:21 smithi174 ceph-mon[53014]: pgmap v374: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-03-26T21:10:21.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:21 smithi063 ceph-mon[66934]: pgmap v374: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-03-26T21:10:23.019 INFO:teuthology.orchestra.run.smithi063.stdout:true 2024-03-26T21:10:23.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:23 smithi174 ceph-mon[53014]: pgmap v375: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-26T21:10:23.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:23 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:10:23.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:23 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:10:23.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:23 smithi063 ceph-mon[66934]: pgmap v375: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-26T21:10:23.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:23 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:10:23.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:23 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:10:23.851 INFO:teuthology.orchestra.run.smithi063.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-26T21:10:23.852 INFO:teuthology.orchestra.run.smithi063.stdout:alertmanager.smithi063 smithi063 *:9093,9094 running (6s) 1s ago 20m 16.4M - 0.25.0 c8568f914cd2 4a33e2cfdb93 2024-03-26T21:10:23.852 INFO:teuthology.orchestra.run.smithi063.stdout:crash.smithi063 smithi063 running (7m) 1s ago 20m 6665k - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 cf1096d84270 2024-03-26T21:10:23.852 INFO:teuthology.orchestra.run.smithi063.stdout:crash.smithi174 smithi174 running (7m) 38s ago 20m 6665k - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 cb5371a92684 2024-03-26T21:10:23.852 INFO:teuthology.orchestra.run.smithi063.stdout:grafana.smithi063 smithi063 *:3000 error 1s ago 20m - - 2024-03-26T21:10:23.852 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foofs.smithi063.tzaznz smithi063 running (110s) 1s ago 14m 32.3M - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 ef76ec94effe 2024-03-26T21:10:23.852 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foofs.smithi174.gitdvk smithi174 running (86s) 38s ago 14m 14.7M - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 1ac021fe2172 2024-03-26T21:10:23.852 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.smithi063.lxhujt smithi063 *:8443,9283,8765 running (10m) 1s ago 21m 518M - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 6d9d10330a90 2024-03-26T21:10:23.852 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.smithi174.sexoin smithi174 *:8443,9283,8765 running (9m) 38s ago 20m 453M - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 d7546233672d 2024-03-26T21:10:23.852 INFO:teuthology.orchestra.run.smithi063.stdout:mon.smithi063 smithi063 running (9m) 1s ago 22m 65.7M 2048M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 c644200344f8 2024-03-26T21:10:23.852 INFO:teuthology.orchestra.run.smithi063.stdout:mon.smithi174 smithi174 running (8m) 38s ago 19m 53.5M 2048M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 1a6d442692e7 2024-03-26T21:10:23.852 INFO:teuthology.orchestra.run.smithi063.stdout:nfs.foo.0.0.smithi063.fgmpwa smithi063 *:2049 running (72s) 1s ago 14m 69.1M - 5.7 a9bdc3eaeb10 5944f2d65c1f 2024-03-26T21:10:23.852 INFO:teuthology.orchestra.run.smithi063.stdout:nfs.foo.1.0.smithi174.pytptt smithi174 *:2049 running (66s) 38s ago 14m 68.5M - 5.7 a9bdc3eaeb10 e604d7340c73 2024-03-26T21:10:23.852 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.smithi063 smithi063 *:9100 running (48s) 1s ago 20m 8006k - 1.5.0 0da6a335fe13 fead24e0db0a 2024-03-26T21:10:23.853 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.smithi174 smithi174 *:9100 running (42s) 38s ago 19m 3586k - 1.5.0 0da6a335fe13 91e70091b852 2024-03-26T21:10:23.853 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0 smithi063 running (7m) 1s ago 19m 419M 1936M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 2d4b3fa825cf 2024-03-26T21:10:23.853 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1 smithi063 running (6m) 1s ago 18m 373M 1936M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 d41fbe507197 2024-03-26T21:10:23.853 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2 smithi063 running (5m) 1s ago 18m 349M 1936M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 f123eeabd3fa 2024-03-26T21:10:23.853 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3 smithi063 running (4m) 1s ago 17m 426M 1936M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 953816eafe87 2024-03-26T21:10:23.853 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4 smithi174 running (4m) 38s ago 17m 360M 2704M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 8aa35bff6cea 2024-03-26T21:10:23.853 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5 smithi174 running (3m) 38s ago 17m 312M 2704M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 25dde01e0676 2024-03-26T21:10:23.853 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6 smithi174 running (2m) 38s ago 16m 301M 2704M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 95fa1b77701b 2024-03-26T21:10:23.853 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7 smithi174 running (2m) 38s ago 16m 294M 2704M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 cb89961bc930 2024-03-26T21:10:23.853 INFO:teuthology.orchestra.run.smithi063.stdout:prometheus.smithi063 smithi063 *:9095 running (24s) 1s ago 19m 50.2M - 2.43.0 a07b618ecd1d ef5097115b20 2024-03-26T21:10:24.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:24 smithi174 ceph-mon[53014]: from='client.34590 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:10:24.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:24 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:10:24.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:24 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:10:24.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:24 smithi063 ceph-mon[66934]: from='client.34590 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:10:24.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:24 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:10:24.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:24 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:10:24.319 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-03-26T21:10:24.319 INFO:teuthology.orchestra.run.smithi063.stdout: "mon": { 2024-03-26T21:10:24.319 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 2 2024-03-26T21:10:24.319 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:10:24.319 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr": { 2024-03-26T21:10:24.319 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 2 2024-03-26T21:10:24.319 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:10:24.319 INFO:teuthology.orchestra.run.smithi063.stdout: "osd": { 2024-03-26T21:10:24.320 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 8 2024-03-26T21:10:24.320 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:10:24.320 INFO:teuthology.orchestra.run.smithi063.stdout: "mds": { 2024-03-26T21:10:24.320 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 2 2024-03-26T21:10:24.320 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:10:24.320 INFO:teuthology.orchestra.run.smithi063.stdout: "overall": { 2024-03-26T21:10:24.320 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 14 2024-03-26T21:10:24.320 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-03-26T21:10:24.320 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-03-26T21:10:24.686 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-03-26T21:10:24.687 INFO:teuthology.orchestra.run.smithi063.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d", 2024-03-26T21:10:24.687 INFO:teuthology.orchestra.run.smithi063.stdout: "in_progress": true, 2024-03-26T21:10:24.687 INFO:teuthology.orchestra.run.smithi063.stdout: "which": "Upgrading all daemon types on all hosts", 2024-03-26T21:10:24.687 INFO:teuthology.orchestra.run.smithi063.stdout: "services_complete": [ 2024-03-26T21:10:24.687 INFO:teuthology.orchestra.run.smithi063.stdout: "nfs", 2024-03-26T21:10:24.687 INFO:teuthology.orchestra.run.smithi063.stdout: "mon", 2024-03-26T21:10:24.687 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr", 2024-03-26T21:10:24.687 INFO:teuthology.orchestra.run.smithi063.stdout: "osd", 2024-03-26T21:10:24.687 INFO:teuthology.orchestra.run.smithi063.stdout: "crash", 2024-03-26T21:10:24.687 INFO:teuthology.orchestra.run.smithi063.stdout: "mds" 2024-03-26T21:10:24.687 INFO:teuthology.orchestra.run.smithi063.stdout: ], 2024-03-26T21:10:24.687 INFO:teuthology.orchestra.run.smithi063.stdout: "progress": "18/23 daemons upgraded", 2024-03-26T21:10:24.687 INFO:teuthology.orchestra.run.smithi063.stdout: "message": "Currently upgrading alertmanager daemons", 2024-03-26T21:10:24.688 INFO:teuthology.orchestra.run.smithi063.stdout: "is_paused": false 2024-03-26T21:10:24.688 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-03-26T21:10:25.115 INFO:teuthology.orchestra.run.smithi063.stdout:HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-26T21:10:25.116 INFO:teuthology.orchestra.run.smithi063.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-26T21:10:25.116 INFO:teuthology.orchestra.run.smithi063.stdout: daemon grafana.smithi063 on smithi063 is in error state 2024-03-26T21:10:25.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:25 smithi174 ceph-mon[53014]: from='client.34594 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:10:25.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:25 smithi174 ceph-mon[53014]: from='client.34598 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:10:25.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:25 smithi174 ceph-mon[53014]: pgmap v376: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:10:25.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:25 smithi174 ceph-mon[53014]: from='client.? 172.21.15.63:0/1206312971' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:25.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:25 smithi063 ceph-mon[66934]: from='client.34594 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:10:25.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:25 smithi063 ceph-mon[66934]: from='client.34598 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:10:25.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:25 smithi063 ceph-mon[66934]: pgmap v376: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:10:25.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:25 smithi063 ceph-mon[66934]: from='client.? 172.21.15.63:0/1206312971' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:26.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:26 smithi174 ceph-mon[53014]: from='client.34606 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:10:26.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:26 smithi174 ceph-mon[53014]: from='client.? 172.21.15.63:0/977198147' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-26T21:10:26.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:26 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:10:26.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:26 smithi063 ceph-mon[66934]: from='client.34606 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:10:26.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:26 smithi063 ceph-mon[66934]: from='client.? 172.21.15.63:0/977198147' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-26T21:10:26.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:26 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:10:27.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:27 smithi063 ceph-mon[66934]: pgmap v377: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:10:27.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:27 smithi174 ceph-mon[53014]: pgmap v377: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:10:28.692 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:28 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:10:28.693 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:28 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:10:28.693 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:28 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:10:28.693 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:28 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T21:10:28.693 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:28 smithi063 ceph-mon[66934]: pgmap v378: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 176 B/s rd, 0 op/s 2024-03-26T21:10:28.693 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:28 smithi063 ceph-mon[66934]: pgmap v379: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 213 B/s rd, 0 op/s 2024-03-26T21:10:28.693 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:28 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:10:28.693 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:28 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:10:28.693 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:28 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-26T21:10:28.693 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:28 smithi063 ceph-mon[66934]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-26T21:10:28.693 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:28 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:10:28.693 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:28 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:28.693 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:28 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:28.693 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:28 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:28.694 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:28 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:28.694 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:28 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:28.694 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:28 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:28.694 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:28 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:28.694 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:28 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:28.694 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:28 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:28.694 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:28 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:28.694 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:28 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:28.694 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:28 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:28.694 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:28 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:28.694 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:28 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:28.694 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:28 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:28.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:28 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:10:28.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:28 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:10:28.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:28 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:10:28.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:28 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T21:10:28.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:28 smithi174 ceph-mon[53014]: pgmap v378: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 176 B/s rd, 0 op/s 2024-03-26T21:10:28.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:28 smithi174 ceph-mon[53014]: pgmap v379: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 213 B/s rd, 0 op/s 2024-03-26T21:10:28.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:28 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:10:28.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:28 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:10:28.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:28 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-26T21:10:28.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:28 smithi174 ceph-mon[53014]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-26T21:10:28.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:28 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:10:28.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:28 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:28.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:28 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:28.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:28 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:28.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:28 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:28.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:28 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:28.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:28 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:28.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:28 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:28.800 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:28 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:28.800 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:28 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:28.800 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:28 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:28.800 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:28 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:28.800 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:28 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:28.800 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:28 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:28.800 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:28 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:28.800 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:28 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:30.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:30 smithi063 ceph-mon[66934]: Upgrade: Updating grafana.smithi063 2024-03-26T21:10:30.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:30 smithi063 ceph-mon[66934]: Deploying daemon grafana.smithi063 on smithi063 2024-03-26T21:10:30.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:30 smithi063 ceph-mon[66934]: pgmap v380: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 213 B/s rd, 0 op/s 2024-03-26T21:10:31.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:30 smithi174 ceph-mon[53014]: Upgrade: Updating grafana.smithi063 2024-03-26T21:10:31.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:30 smithi174 ceph-mon[53014]: Deploying daemon grafana.smithi063 on smithi063 2024-03-26T21:10:31.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:30 smithi174 ceph-mon[53014]: pgmap v380: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 213 B/s rd, 0 op/s 2024-03-26T21:10:32.995 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:32 smithi063 ceph-mon[66934]: pgmap v381: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 213 B/s rd, 0 op/s 2024-03-26T21:10:33.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:32 smithi174 ceph-mon[53014]: pgmap v381: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 213 B/s rd, 0 op/s 2024-03-26T21:10:34.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:34 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:10:34.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:34 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:10:34.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:34 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:10:34.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:34 smithi174 ceph-mon[53014]: pgmap v382: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 213 B/s rd, 0 op/s 2024-03-26T21:10:34.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:34 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:10:34.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:34 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:10:34.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:34 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:10:34.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:34 smithi063 ceph-mon[66934]: pgmap v382: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 213 B/s rd, 0 op/s 2024-03-26T21:10:36.870 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:36 smithi063 ceph-mon[66934]: pgmap v383: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 213 B/s rd, 0 op/s 2024-03-26T21:10:37.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:36 smithi174 ceph-mon[53014]: pgmap v383: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 213 B/s rd, 0 op/s 2024-03-26T21:10:39.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:38 smithi174 ceph-mon[53014]: pgmap v384: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-03-26T21:10:39.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:38 smithi063 ceph-mon[66934]: pgmap v384: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-03-26T21:10:40.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:39 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:10:40.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:39 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:10:40.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:39 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:10:40.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:39 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:10:41.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:40 smithi063 ceph-mon[66934]: pgmap v385: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:10:41.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:40 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:10:41.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:40 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:10:41.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:40 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:10:41.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:40 smithi174 ceph-mon[53014]: pgmap v385: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:10:41.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:40 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:10:41.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:40 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:10:41.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:40 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:10:43.194 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:42 smithi063 ceph-mon[66934]: pgmap v386: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-26T21:10:43.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:42 smithi174 ceph-mon[53014]: pgmap v386: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-26T21:10:45.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:44 smithi174 ceph-mon[53014]: pgmap v387: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:10:45.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:44 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:10:45.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:44 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:10:45.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:44 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:10:45.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:44 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T21:10:45.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:44 smithi174 ceph-mon[53014]: pgmap v388: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 203 B/s rd, 0 op/s 2024-03-26T21:10:45.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:44 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:10:45.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:44 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:10:45.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:44 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-26T21:10:45.050 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:44 smithi174 ceph-mon[53014]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-26T21:10:45.050 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:44 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:10:45.050 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:44 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:45.050 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:44 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:45.050 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:44 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:45.050 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:44 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:45.050 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:44 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:45.050 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:44 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:45.050 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:44 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:45.050 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:44 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:45.050 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:44 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:45.050 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:44 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:45.051 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:44 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:45.051 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:44 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:45.051 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:44 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:45.051 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:44 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:45.051 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:44 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:45.051 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:44 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:45.051 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:44 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:45.051 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:44 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:45.051 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:44 smithi174 ceph-mon[53014]: Upgrade: Finalizing container_image settings 2024-03-26T21:10:45.051 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:44 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:10:45.051 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:44 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-03-26T21:10:45.051 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:44 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-03-26T21:10:45.051 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:44 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-03-26T21:10:45.051 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:44 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-03-26T21:10:45.052 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:44 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-03-26T21:10:45.052 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:44 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-03-26T21:10:45.052 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:44 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-03-26T21:10:45.052 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:44 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-03-26T21:10:45.052 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:44 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-03-26T21:10:45.052 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:44 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-03-26T21:10:45.052 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:44 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-03-26T21:10:45.052 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:44 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-03-26T21:10:45.052 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:44 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-03-26T21:10:45.052 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:44 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-03-26T21:10:45.052 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:44 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-03-26T21:10:45.052 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:44 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-03-26T21:10:45.052 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:44 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-03-26T21:10:45.053 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:44 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-03-26T21:10:45.053 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:44 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-03-26T21:10:45.053 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:44 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-03-26T21:10:45.053 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:44 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-03-26T21:10:45.053 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:44 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-03-26T21:10:45.053 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:44 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-03-26T21:10:45.053 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:44 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-03-26T21:10:45.053 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:44 smithi174 ceph-mon[53014]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-03-26T21:10:45.053 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:44 smithi174 ceph-mon[53014]: Cluster is now healthy 2024-03-26T21:10:45.053 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:44 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-03-26T21:10:45.053 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:44 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-03-26T21:10:45.053 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:44 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-03-26T21:10:45.053 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:44 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-03-26T21:10:45.053 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:44 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-03-26T21:10:45.053 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:44 smithi174 ceph-mon[53014]: Upgrade: Complete! 2024-03-26T21:10:45.054 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:44 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-03-26T21:10:45.054 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:44 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-03-26T21:10:45.054 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:44 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:10:45.054 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:44 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:10:45.054 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:44 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T21:10:45.054 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:44 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:10:45.054 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:44 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:10:45.054 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:44 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:10:45.054 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:44 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:10:45.054 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:44 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T21:10:45.054 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:44 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:10:45.054 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:44 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:10:45.055 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:44 smithi063 ceph-mon[66934]: pgmap v387: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:10:45.055 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:44 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:10:45.055 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:44 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:10:45.055 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:44 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:10:45.055 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:44 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T21:10:45.055 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:44 smithi063 ceph-mon[66934]: pgmap v388: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 203 B/s rd, 0 op/s 2024-03-26T21:10:45.056 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:44 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:10:45.056 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:44 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:10:45.056 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:44 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-26T21:10:45.056 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:44 smithi063 ceph-mon[66934]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-26T21:10:45.056 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:44 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:10:45.056 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:44 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:45.056 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:44 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:45.056 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:44 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:45.056 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:44 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:45.056 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:44 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:45.056 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:44 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:45.056 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:44 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:45.056 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:44 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:45.057 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:44 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:45.057 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:44 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:45.057 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:44 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:45.057 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:44 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:45.057 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:44 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:45.057 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:44 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:45.057 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:44 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:45.057 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:44 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:45.057 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:44 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:45.057 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:44 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:10:45.057 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:44 smithi063 ceph-mon[66934]: Upgrade: Finalizing container_image settings 2024-03-26T21:10:45.057 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:44 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:10:45.057 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:44 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-03-26T21:10:45.057 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:44 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-03-26T21:10:45.058 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:44 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-03-26T21:10:45.058 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:44 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-03-26T21:10:45.058 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:44 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-03-26T21:10:45.058 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:44 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-03-26T21:10:45.058 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:44 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-03-26T21:10:45.058 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:44 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-03-26T21:10:45.058 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:44 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-03-26T21:10:45.059 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:44 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-03-26T21:10:45.059 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:44 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-03-26T21:10:45.059 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:44 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-03-26T21:10:45.059 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:44 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-03-26T21:10:45.059 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:44 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-03-26T21:10:45.059 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:44 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-03-26T21:10:45.059 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:44 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-03-26T21:10:45.059 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:44 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-03-26T21:10:45.059 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:44 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-03-26T21:10:45.059 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:44 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-03-26T21:10:45.059 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:44 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-03-26T21:10:45.059 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:44 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-03-26T21:10:45.059 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:44 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-03-26T21:10:45.059 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:44 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-03-26T21:10:45.060 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:44 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-03-26T21:10:45.060 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:44 smithi063 ceph-mon[66934]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-03-26T21:10:45.060 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:44 smithi063 ceph-mon[66934]: Cluster is now healthy 2024-03-26T21:10:45.060 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:44 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-03-26T21:10:45.060 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:44 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-03-26T21:10:45.060 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:44 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-03-26T21:10:45.060 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:44 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-03-26T21:10:45.060 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:44 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-03-26T21:10:45.060 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:44 smithi063 ceph-mon[66934]: Upgrade: Complete! 2024-03-26T21:10:45.060 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:44 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-03-26T21:10:45.060 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:44 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-03-26T21:10:45.060 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:44 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:10:45.060 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:44 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:10:45.060 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:44 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T21:10:45.061 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:44 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:10:45.061 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:44 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:10:45.061 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:44 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:10:45.061 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:44 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:10:45.061 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:44 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T21:10:45.061 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:44 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:10:45.061 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:44 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:10:47.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:47 smithi174 ceph-mon[53014]: pgmap v389: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 203 B/s rd, 0 op/s 2024-03-26T21:10:47.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:47 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:10:47.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:47 smithi063 ceph-mon[66934]: pgmap v389: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 203 B/s rd, 0 op/s 2024-03-26T21:10:47.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:47 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:10:49.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:49 smithi174 ceph-mon[53014]: pgmap v390: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 203 B/s rd, 0 op/s 2024-03-26T21:10:49.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:49 smithi063 ceph-mon[66934]: pgmap v390: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 203 B/s rd, 0 op/s 2024-03-26T21:10:51.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:51 smithi174 ceph-mon[53014]: pgmap v391: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 203 B/s rd, 0 op/s 2024-03-26T21:10:51.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:51 smithi063 ceph-mon[66934]: pgmap v391: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 203 B/s rd, 0 op/s 2024-03-26T21:10:53.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:53 smithi174 ceph-mon[53014]: pgmap v392: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 203 B/s rd, 0 op/s 2024-03-26T21:10:53.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:53 smithi063 ceph-mon[66934]: pgmap v392: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 203 B/s rd, 0 op/s 2024-03-26T21:10:55.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:55 smithi174 ceph-mon[53014]: pgmap v393: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 203 B/s rd, 0 op/s 2024-03-26T21:10:55.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:55 smithi063 ceph-mon[66934]: pgmap v393: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 203 B/s rd, 0 op/s 2024-03-26T21:10:56.188 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -e sha1=42fc52b86b14d4f88c40d75eb7102e0ab3918f55 -- bash -c 'ceph orch ps' 2024-03-26T21:10:56.752 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:56 smithi063 ceph-mon[66934]: from='client.34614 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:10:56.752 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:56 smithi063 ceph-mon[66934]: pgmap v394: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:10:56.752 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:56 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:10:56.752 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:56 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:10:57.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:56 smithi174 ceph-mon[53014]: from='client.34614 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:10:57.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:56 smithi174 ceph-mon[53014]: pgmap v394: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:10:57.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:56 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:10:57.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:56 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:10:58.195 INFO:teuthology.orchestra.run.smithi063.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-26T21:10:58.195 INFO:teuthology.orchestra.run.smithi063.stdout:alertmanager.smithi063 smithi063 *:9093,9094 running (41s) 19s ago 21m 16.4M - 0.25.0 c8568f914cd2 4a33e2cfdb93 2024-03-26T21:10:58.195 INFO:teuthology.orchestra.run.smithi063.stdout:crash.smithi063 smithi063 running (8m) 19s ago 21m 6665k - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 cf1096d84270 2024-03-26T21:10:58.195 INFO:teuthology.orchestra.run.smithi063.stdout:crash.smithi174 smithi174 running (8m) 73s ago 20m 6665k - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 cb5371a92684 2024-03-26T21:10:58.195 INFO:teuthology.orchestra.run.smithi063.stdout:grafana.smithi063 smithi063 *:3000 running (25s) 19s ago 21m 47.6M - 9.4.12 2bacad6d85d8 b95ec13af4b7 2024-03-26T21:10:58.195 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foofs.smithi063.tzaznz smithi063 running (2m) 19s ago 15m 32.4M - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 ef76ec94effe 2024-03-26T21:10:58.195 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foofs.smithi174.gitdvk smithi174 running (2m) 73s ago 15m 14.7M - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 1ac021fe2172 2024-03-26T21:10:58.195 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.smithi063.lxhujt smithi063 *:8443,9283,8765 running (11m) 19s ago 22m 525M - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 6d9d10330a90 2024-03-26T21:10:58.195 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.smithi174.sexoin smithi174 *:8443,9283,8765 running (10m) 73s ago 20m 453M - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 d7546233672d 2024-03-26T21:10:58.195 INFO:teuthology.orchestra.run.smithi063.stdout:mon.smithi063 smithi063 running (9m) 19s ago 22m 66.5M 2048M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 c644200344f8 2024-03-26T21:10:58.195 INFO:teuthology.orchestra.run.smithi063.stdout:mon.smithi174 smithi174 running (9m) 73s ago 20m 53.5M 2048M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 1a6d442692e7 2024-03-26T21:10:58.195 INFO:teuthology.orchestra.run.smithi063.stdout:nfs.foo.0.0.smithi063.fgmpwa smithi063 *:2049 running (106s) 19s ago 15m 69.2M - 5.7 a9bdc3eaeb10 5944f2d65c1f 2024-03-26T21:10:58.196 INFO:teuthology.orchestra.run.smithi063.stdout:nfs.foo.1.0.smithi174.pytptt smithi174 *:2049 running (100s) 73s ago 14m 68.5M - 5.7 a9bdc3eaeb10 e604d7340c73 2024-03-26T21:10:58.196 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.smithi063 smithi063 *:9100 running (83s) 19s ago 20m 8358k - 1.5.0 0da6a335fe13 fead24e0db0a 2024-03-26T21:10:58.196 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.smithi174 smithi174 *:9100 running (76s) 73s ago 20m 3586k - 1.5.0 0da6a335fe13 91e70091b852 2024-03-26T21:10:58.196 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0 smithi063 running (7m) 19s ago 19m 420M 1936M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 2d4b3fa825cf 2024-03-26T21:10:58.196 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1 smithi063 running (6m) 19s ago 19m 373M 1936M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 d41fbe507197 2024-03-26T21:10:58.196 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2 smithi063 running (6m) 19s ago 18m 349M 1936M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 f123eeabd3fa 2024-03-26T21:10:58.196 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3 smithi063 running (5m) 19s ago 18m 427M 1936M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 953816eafe87 2024-03-26T21:10:58.196 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4 smithi174 running (4m) 73s ago 18m 360M 2704M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 8aa35bff6cea 2024-03-26T21:10:58.196 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5 smithi174 running (4m) 73s ago 17m 312M 2704M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 25dde01e0676 2024-03-26T21:10:58.196 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6 smithi174 running (3m) 73s ago 17m 301M 2704M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 95fa1b77701b 2024-03-26T21:10:58.196 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7 smithi174 running (2m) 73s ago 16m 294M 2704M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 cb89961bc930 2024-03-26T21:10:58.196 INFO:teuthology.orchestra.run.smithi063.stdout:prometheus.smithi063 smithi063 *:9095 running (58s) 19s ago 20m 53.1M - 2.43.0 a07b618ecd1d ef5097115b20 2024-03-26T21:10:58.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:58 smithi063 ceph-mon[66934]: pgmap v395: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:10:58.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:10:58 smithi063 ceph-mon[66934]: from='client.34618 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:10:58.826 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -e sha1=42fc52b86b14d4f88c40d75eb7102e0ab3918f55 -- bash -c 'ceph versions' 2024-03-26T21:10:59.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:58 smithi174 ceph-mon[53014]: pgmap v395: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:10:59.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:10:58 smithi174 ceph-mon[53014]: from='client.34618 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:11:00.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:11:00 smithi063 ceph-mon[66934]: pgmap v396: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:11:00.838 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-03-26T21:11:00.838 INFO:teuthology.orchestra.run.smithi063.stdout: "mon": { 2024-03-26T21:11:00.838 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 2 2024-03-26T21:11:00.838 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:11:00.838 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr": { 2024-03-26T21:11:00.838 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 2 2024-03-26T21:11:00.838 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:11:00.838 INFO:teuthology.orchestra.run.smithi063.stdout: "osd": { 2024-03-26T21:11:00.839 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 8 2024-03-26T21:11:00.839 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:11:00.839 INFO:teuthology.orchestra.run.smithi063.stdout: "mds": { 2024-03-26T21:11:00.839 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 2 2024-03-26T21:11:00.839 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:11:00.839 INFO:teuthology.orchestra.run.smithi063.stdout: "overall": { 2024-03-26T21:11:00.839 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 14 2024-03-26T21:11:00.839 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-03-26T21:11:00.839 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-03-26T21:11:01.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:11:00 smithi174 ceph-mon[53014]: pgmap v396: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:11:01.498 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -e sha1=42fc52b86b14d4f88c40d75eb7102e0ab3918f55 -- bash -c 'echo "wait for servicemap items w/ changing names to refresh"' 2024-03-26T21:11:01.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:11:01 smithi063 ceph-mon[66934]: from='client.? 172.21.15.63:0/3180984108' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:11:01.949 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:11:01 smithi174 ceph-mon[53014]: from='client.? 172.21.15.63:0/3180984108' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:11:02.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:11:02 smithi063 ceph-mon[66934]: pgmap v397: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-26T21:11:03.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:11:02 smithi174 ceph-mon[53014]: pgmap v397: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-26T21:11:03.293 INFO:teuthology.orchestra.run.smithi063.stdout:wait for servicemap items w/ changing names to refresh 2024-03-26T21:11:03.956 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -e sha1=42fc52b86b14d4f88c40d75eb7102e0ab3918f55 -- bash -c 'sleep 60' 2024-03-26T21:11:04.945 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:11:04 smithi063 ceph-mon[66934]: pgmap v398: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:11:05.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:11:04 smithi174 ceph-mon[53014]: pgmap v398: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:11:07.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:11:06 smithi174 ceph-mon[53014]: pgmap v399: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:11:07.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:11:06 smithi063 ceph-mon[66934]: pgmap v399: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:11:09.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:11:08 smithi174 ceph-mon[53014]: pgmap v400: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:11:09.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:11:08 smithi063 ceph-mon[66934]: pgmap v400: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:11:11.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:11:10 smithi174 ceph-mon[53014]: pgmap v401: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:11:11.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:11:10 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:11:11.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:11:10 smithi063 ceph-mon[66934]: pgmap v401: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:11:11.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:11:10 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:11:13.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:11:12 smithi174 ceph-mon[53014]: pgmap v402: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-26T21:11:13.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:11:12 smithi063 ceph-mon[66934]: pgmap v402: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-26T21:11:15.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:11:14 smithi174 ceph-mon[53014]: pgmap v403: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:11:15.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:11:14 smithi063 ceph-mon[66934]: pgmap v403: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:11:17.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:11:16 smithi174 ceph-mon[53014]: pgmap v404: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:11:17.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:11:16 smithi063 ceph-mon[66934]: pgmap v404: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:11:19.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:11:18 smithi174 ceph-mon[53014]: pgmap v405: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:11:19.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:11:18 smithi063 ceph-mon[66934]: pgmap v405: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:11:21.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:11:20 smithi174 ceph-mon[53014]: pgmap v406: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:11:21.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:11:20 smithi063 ceph-mon[66934]: pgmap v406: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:11:23.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:11:22 smithi174 ceph-mon[53014]: pgmap v407: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-26T21:11:23.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:11:22 smithi063 ceph-mon[66934]: pgmap v407: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-26T21:11:25.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:11:24 smithi174 ceph-mon[53014]: pgmap v408: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:11:25.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:11:24 smithi063 ceph-mon[66934]: pgmap v408: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:11:26.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:11:25 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:11:26.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:11:25 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:11:27.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:11:26 smithi174 ceph-mon[53014]: pgmap v409: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:11:27.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:11:26 smithi063 ceph-mon[66934]: pgmap v409: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:11:29.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:11:28 smithi174 ceph-mon[53014]: pgmap v410: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:11:29.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:11:28 smithi063 ceph-mon[66934]: pgmap v410: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:11:31.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:11:30 smithi174 ceph-mon[53014]: pgmap v411: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:11:31.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:11:30 smithi063 ceph-mon[66934]: pgmap v411: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:11:33.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:11:32 smithi174 ceph-mon[53014]: pgmap v412: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-26T21:11:33.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:11:32 smithi063 ceph-mon[66934]: pgmap v412: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-26T21:11:35.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:11:34 smithi174 ceph-mon[53014]: pgmap v413: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:11:35.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:11:34 smithi063 ceph-mon[66934]: pgmap v413: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:11:37.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:11:36 smithi174 ceph-mon[53014]: pgmap v414: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:11:37.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:11:36 smithi063 ceph-mon[66934]: pgmap v414: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:11:39.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:11:38 smithi174 ceph-mon[53014]: pgmap v415: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:11:39.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:11:38 smithi063 ceph-mon[66934]: pgmap v415: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:11:41.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:11:40 smithi174 ceph-mon[53014]: pgmap v416: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:11:41.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:11:40 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:11:41.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:11:40 smithi063 ceph-mon[66934]: pgmap v416: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:11:41.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:11:40 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:11:43.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:11:42 smithi174 ceph-mon[53014]: pgmap v417: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-26T21:11:43.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:11:42 smithi063 ceph-mon[66934]: pgmap v417: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-26T21:11:45.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:11:44 smithi174 ceph-mon[53014]: pgmap v418: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:11:45.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:11:44 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:11:45.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:11:44 smithi063 ceph-mon[66934]: pgmap v418: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:11:45.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:11:44 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-26T21:11:46.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:11:45 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:11:46.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:11:45 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T21:11:46.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:11:45 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:11:46.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:11:45 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:11:46.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:11:45 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-26T21:11:46.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:11:45 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-26T21:11:46.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:11:45 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:11:46.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:11:45 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' 2024-03-26T21:11:47.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:11:46 smithi174 ceph-mon[53014]: pgmap v419: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:11:47.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:11:46 smithi063 ceph-mon[66934]: pgmap v419: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:11:49.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:11:48 smithi174 ceph-mon[53014]: pgmap v420: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:11:49.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:11:48 smithi063 ceph-mon[66934]: pgmap v420: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:11:51.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:11:50 smithi174 ceph-mon[53014]: pgmap v421: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:11:51.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:11:50 smithi063 ceph-mon[66934]: pgmap v421: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:11:53.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:11:52 smithi174 ceph-mon[53014]: pgmap v422: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-26T21:11:53.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:11:52 smithi063 ceph-mon[66934]: pgmap v422: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-26T21:11:55.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:11:54 smithi174 ceph-mon[53014]: pgmap v423: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:11:55.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:11:54 smithi063 ceph-mon[66934]: pgmap v423: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:11:56.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:11:55 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:11:56.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:11:55 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:11:57.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:11:56 smithi174 ceph-mon[53014]: pgmap v424: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:11:57.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:11:56 smithi063 ceph-mon[66934]: pgmap v424: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:11:59.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:11:58 smithi174 ceph-mon[53014]: pgmap v425: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:11:59.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:11:58 smithi063 ceph-mon[66934]: pgmap v425: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:12:01.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:12:00 smithi174 ceph-mon[53014]: pgmap v426: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:12:01.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:12:00 smithi063 ceph-mon[66934]: pgmap v426: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:12:03.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:12:02 smithi174 ceph-mon[53014]: pgmap v427: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-26T21:12:03.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:12:02 smithi063 ceph-mon[66934]: pgmap v427: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-26T21:12:05.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:12:04 smithi174 ceph-mon[53014]: pgmap v428: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:12:05.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:12:04 smithi063 ceph-mon[66934]: pgmap v428: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:12:06.389 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -e sha1=42fc52b86b14d4f88c40d75eb7102e0ab3918f55 -- bash -c 'ceph orch ps' 2024-03-26T21:12:07.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:12:06 smithi063 ceph-mon[66934]: pgmap v429: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:12:07.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:12:06 smithi174 ceph-mon[53014]: pgmap v429: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:12:08.425 INFO:teuthology.orchestra.run.smithi063.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-26T21:12:08.425 INFO:teuthology.orchestra.run.smithi063.stdout:alertmanager.smithi063 smithi063 *:9093,9094 running (111s) 89s ago 22m 16.4M - 0.25.0 c8568f914cd2 4a33e2cfdb93 2024-03-26T21:12:08.425 INFO:teuthology.orchestra.run.smithi063.stdout:crash.smithi063 smithi063 running (9m) 89s ago 22m 6665k - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 cf1096d84270 2024-03-26T21:12:08.425 INFO:teuthology.orchestra.run.smithi063.stdout:crash.smithi174 smithi174 running (9m) 2m ago 21m 6665k - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 cb5371a92684 2024-03-26T21:12:08.425 INFO:teuthology.orchestra.run.smithi063.stdout:grafana.smithi063 smithi063 *:3000 running (95s) 89s ago 22m 47.6M - 9.4.12 2bacad6d85d8 b95ec13af4b7 2024-03-26T21:12:08.425 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foofs.smithi063.tzaznz smithi063 running (3m) 89s ago 16m 32.4M - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 ef76ec94effe 2024-03-26T21:12:08.425 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foofs.smithi174.gitdvk smithi174 running (3m) 2m ago 16m 14.7M - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 1ac021fe2172 2024-03-26T21:12:08.425 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.smithi063.lxhujt smithi063 *:8443,9283,8765 running (12m) 89s ago 23m 525M - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 6d9d10330a90 2024-03-26T21:12:08.425 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.smithi174.sexoin smithi174 *:8443,9283,8765 running (11m) 2m ago 21m 453M - 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 d7546233672d 2024-03-26T21:12:08.426 INFO:teuthology.orchestra.run.smithi063.stdout:mon.smithi063 smithi063 running (11m) 89s ago 23m 66.5M 2048M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 c644200344f8 2024-03-26T21:12:08.426 INFO:teuthology.orchestra.run.smithi063.stdout:mon.smithi174 smithi174 running (10m) 2m ago 21m 53.5M 2048M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 1a6d442692e7 2024-03-26T21:12:08.426 INFO:teuthology.orchestra.run.smithi063.stdout:nfs.foo.0.0.smithi063.fgmpwa smithi063 *:2049 running (2m) 89s ago 16m 69.2M - 5.7 a9bdc3eaeb10 5944f2d65c1f 2024-03-26T21:12:08.426 INFO:teuthology.orchestra.run.smithi063.stdout:nfs.foo.1.0.smithi174.pytptt smithi174 *:2049 running (2m) 2m ago 16m 68.5M - 5.7 a9bdc3eaeb10 e604d7340c73 2024-03-26T21:12:08.426 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.smithi063 smithi063 *:9100 running (2m) 89s ago 22m 8358k - 1.5.0 0da6a335fe13 fead24e0db0a 2024-03-26T21:12:08.426 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.smithi174 smithi174 *:9100 running (2m) 2m ago 21m 3586k - 1.5.0 0da6a335fe13 91e70091b852 2024-03-26T21:12:08.426 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0 smithi063 running (8m) 89s ago 21m 420M 1936M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 2d4b3fa825cf 2024-03-26T21:12:08.426 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1 smithi063 running (7m) 89s ago 20m 373M 1936M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 d41fbe507197 2024-03-26T21:12:08.426 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2 smithi063 running (7m) 89s ago 20m 349M 1936M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 f123eeabd3fa 2024-03-26T21:12:08.426 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3 smithi063 running (6m) 89s ago 19m 427M 1936M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 953816eafe87 2024-03-26T21:12:08.426 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4 smithi174 running (5m) 2m ago 19m 360M 2704M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 8aa35bff6cea 2024-03-26T21:12:08.426 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5 smithi174 running (5m) 2m ago 18m 312M 2704M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 25dde01e0676 2024-03-26T21:12:08.427 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6 smithi174 running (4m) 2m ago 18m 301M 2704M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 95fa1b77701b 2024-03-26T21:12:08.427 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7 smithi174 running (4m) 2m ago 17m 294M 2704M 19.0.0-2484-g42fc52b8 a9bdc3eaeb10 cb89961bc930 2024-03-26T21:12:08.427 INFO:teuthology.orchestra.run.smithi063.stdout:prometheus.smithi063 smithi063 *:9095 running (2m) 89s ago 21m 53.1M - 2.43.0 a07b618ecd1d ef5097115b20 2024-03-26T21:12:09.095 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -e sha1=42fc52b86b14d4f88c40d75eb7102e0ab3918f55 -- bash -c 'ceph orch upgrade status' 2024-03-26T21:12:09.120 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:12:08 smithi063 ceph-mon[66934]: pgmap v430: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:12:09.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:12:08 smithi174 ceph-mon[53014]: pgmap v430: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:12:10.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:12:09 smithi063 ceph-mon[66934]: from='client.34626 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:12:10.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:12:09 smithi174 ceph-mon[53014]: from='client.34626 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:12:11.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:12:10 smithi063 ceph-mon[66934]: pgmap v431: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:12:11.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:12:10 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:12:11.174 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-03-26T21:12:11.174 INFO:teuthology.orchestra.run.smithi063.stdout: "target_image": null, 2024-03-26T21:12:11.174 INFO:teuthology.orchestra.run.smithi063.stdout: "in_progress": false, 2024-03-26T21:12:11.174 INFO:teuthology.orchestra.run.smithi063.stdout: "which": "", 2024-03-26T21:12:11.174 INFO:teuthology.orchestra.run.smithi063.stdout: "services_complete": [], 2024-03-26T21:12:11.174 INFO:teuthology.orchestra.run.smithi063.stdout: "progress": null, 2024-03-26T21:12:11.174 INFO:teuthology.orchestra.run.smithi063.stdout: "message": "", 2024-03-26T21:12:11.174 INFO:teuthology.orchestra.run.smithi063.stdout: "is_paused": false 2024-03-26T21:12:11.174 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-03-26T21:12:11.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:12:10 smithi174 ceph-mon[53014]: pgmap v431: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:12:11.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:12:10 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:12:11.867 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -e sha1=42fc52b86b14d4f88c40d75eb7102e0ab3918f55 -- bash -c 'ceph health detail' 2024-03-26T21:12:12.233 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:12:11 smithi063 ceph-mon[66934]: from='client.34630 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:12:12.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:12:11 smithi174 ceph-mon[53014]: from='client.34630 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:12:13.128 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:12:12 smithi063 ceph-mon[66934]: pgmap v432: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-26T21:12:13.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:12:12 smithi174 ceph-mon[53014]: pgmap v432: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-26T21:12:13.979 INFO:teuthology.orchestra.run.smithi063.stdout:HEALTH_OK 2024-03-26T21:12:14.630 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -e sha1=42fc52b86b14d4f88c40d75eb7102e0ab3918f55 -- bash -c 'ceph versions' 2024-03-26T21:12:14.943 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:12:14 smithi063 ceph-mon[66934]: pgmap v433: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:12:14.943 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:12:14 smithi063 ceph-mon[66934]: from='client.? 172.21.15.63:0/1999018469' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-26T21:12:15.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:12:14 smithi174 ceph-mon[53014]: pgmap v433: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:12:15.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:12:14 smithi174 ceph-mon[53014]: from='client.? 172.21.15.63:0/1999018469' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-03-26T21:12:16.780 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-03-26T21:12:16.781 INFO:teuthology.orchestra.run.smithi063.stdout: "mon": { 2024-03-26T21:12:16.781 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 2 2024-03-26T21:12:16.781 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:12:16.781 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr": { 2024-03-26T21:12:16.781 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 2 2024-03-26T21:12:16.781 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:12:16.781 INFO:teuthology.orchestra.run.smithi063.stdout: "osd": { 2024-03-26T21:12:16.781 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 8 2024-03-26T21:12:16.781 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:12:16.781 INFO:teuthology.orchestra.run.smithi063.stdout: "mds": { 2024-03-26T21:12:16.781 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 2 2024-03-26T21:12:16.781 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:12:16.782 INFO:teuthology.orchestra.run.smithi063.stdout: "overall": { 2024-03-26T21:12:16.782 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)": 14 2024-03-26T21:12:16.782 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-03-26T21:12:16.782 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-03-26T21:12:17.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:12:16 smithi063 ceph-mon[66934]: pgmap v434: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:12:17.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:12:16 smithi063 ceph-mon[66934]: from='client.? 172.21.15.63:0/3350918286' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:12:17.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:12:16 smithi174 ceph-mon[53014]: pgmap v434: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:12:17.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:12:16 smithi174 ceph-mon[53014]: from='client.? 172.21.15.63:0/3350918286' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:12:17.384 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -e sha1=42fc52b86b14d4f88c40d75eb7102e0ab3918f55 -- bash -c 'ceph versions | jq -e '"'"'.overall | length == 1'"'"'' 2024-03-26T21:12:18.993 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:12:18 smithi063 ceph-mon[66934]: pgmap v435: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:12:19.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:12:18 smithi174 ceph-mon[53014]: pgmap v435: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:12:19.516 INFO:teuthology.orchestra.run.smithi063.stdout:true 2024-03-26T21:12:20.117 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:12:19 smithi063 ceph-mon[66934]: from='client.? 172.21.15.63:0/1653902454' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:12:20.139 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -e sha1=42fc52b86b14d4f88c40d75eb7102e0ab3918f55 -- bash -c 'ceph versions | jq -e '"'"'.overall | keys'"'"' | grep $sha1' 2024-03-26T21:12:20.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:12:19 smithi174 ceph-mon[53014]: from='client.? 172.21.15.63:0/1653902454' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:12:21.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:12:20 smithi063 ceph-mon[66934]: pgmap v436: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:12:21.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:12:20 smithi174 ceph-mon[53014]: pgmap v436: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:12:22.351 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-2484-g42fc52b8 (42fc52b86b14d4f88c40d75eb7102e0ab3918f55) squid (dev)" 2024-03-26T21:12:22.947 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:12:22 smithi063 ceph-mon[66934]: pgmap v437: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-26T21:12:22.947 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:12:22 smithi063 ceph-mon[66934]: from='client.? 172.21.15.63:0/4006272681' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:12:22.969 INFO:teuthology.task.sequential:In sequential, running task cephadm.wait_for_service... 2024-03-26T21:12:22.970 INFO:tasks.cephadm:Waiting for ceph service nfs.foo to start (timeout 300)... 2024-03-26T21:12:22.970 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -- ceph orch ls -f json 2024-03-26T21:12:23.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:12:22 smithi174 ceph-mon[53014]: pgmap v437: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-26T21:12:23.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:12:22 smithi174 ceph-mon[53014]: from='client.? 172.21.15.63:0/4006272681' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-26T21:12:25.075 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-03-26T21:12:25.075 INFO:teuthology.orchestra.run.smithi063.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-03-26T20:49:01.786722Z", "last_refresh": "2024-03-26T21:10:38.745585Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-03-26T20:48:56.123811Z", "last_refresh": "2024-03-26T21:09:45.135646Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "spec": {"anonymous_access": true, "protocol": "https"}, "status": {"created": "2024-03-26T20:48:58.928241Z", "last_refresh": "2024-03-26T21:10:38.745758Z", "ports": [3000], "running": 1, "size": 1}}, {"placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-03-26T20:55:32.409168Z", "last_refresh": "2024-03-26T21:09:45.136774Z", "running": 2, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-03-26T20:48:54.137895Z", "last_refresh": "2024-03-26T21:09:45.135807Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi063:172.21.15.63=smithi063", "smithi174:172.21.15.174=smithi174"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-03-26T20:49:47.043046Z", "last_refresh": "2024-03-26T21:09:45.135946Z", "running": 2, "size": 2}}, {"events": ["2024-03-26T21:11:44.844184Z service:nfs.foo [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foo", "service_name": "nfs.foo", "service_type": "nfs", "spec": {"port": 2049}, "status": {"created": "2024-03-26T20:55:50.414394Z", "last_refresh": "2024-03-26T21:09:45.136908Z", "ports": [2049], "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-03-26T20:49:00.204434Z", "last_refresh": "2024-03-26T21:09:45.136082Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "a9bdc3eaeb106993bcbbac35d7fe5c7b76c052021676e129f5984aed30aee063", "container_image_name": "quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d", "last_refresh": "2024-03-26T21:09:45.136221Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-03-26T20:48:57.277368Z", "last_refresh": "2024-03-26T21:10:38.745913Z", "ports": [9095], "running": 1, "size": 1}}] 2024-03-26T21:12:25.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:12:24 smithi174 ceph-mon[53014]: pgmap v438: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:12:25.301 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:12:24 smithi063 ceph-mon[66934]: pgmap v438: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:12:25.728 INFO:tasks.cephadm:nfs.foo has 2/2 2024-03-26T21:12:25.728 DEBUG:teuthology.parallel:result is None 2024-03-26T21:12:25.728 INFO:teuthology.run_tasks:Running task vip.exec... 2024-03-26T21:12:25.740 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi063.front.sepia.ceph.com 2024-03-26T21:12:25.740 DEBUG:teuthology.orchestra.run.smithi063:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'umount /mnt/foo' 2024-03-26T21:12:25.767 INFO:teuthology.orchestra.run.smithi063.stderr:+ umount /mnt/foo 2024-03-26T21:12:25.778 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-03-26T21:12:25.809 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi063.front.sepia.ceph.com 2024-03-26T21:12:25.809 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -- bash -c 'ceph nfs cluster ls | grep foo' 2024-03-26T21:12:26.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:12:25 smithi063 ceph-mon[66934]: from='client.34650 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-26T21:12:26.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:12:25 smithi063 ceph-mon[66934]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:12:26.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:12:25 smithi174 ceph-mon[53014]: from='client.34650 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-26T21:12:26.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:12:25 smithi174 ceph-mon[53014]: from='mgr.34108 172.21.15.63:0/2232872849' entity='mgr.smithi063.lxhujt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-26T21:12:27.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:12:26 smithi063 ceph-mon[66934]: pgmap v439: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:12:27.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:12:26 smithi174 ceph-mon[53014]: pgmap v439: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:12:27.962 INFO:teuthology.orchestra.run.smithi063.stdout: "foo" 2024-03-26T21:12:28.597 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -- bash -c 'ceph nfs export ls foo --detailed' 2024-03-26T21:12:29.160 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:12:28 smithi063 ceph-mon[66934]: pgmap v440: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:12:29.160 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:12:28 smithi063 ceph-mon[66934]: from='client.34654 -' entity='client.admin' cmd=[{"prefix": "nfs cluster ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:12:29.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:12:28 smithi174 ceph-mon[53014]: pgmap v440: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:12:29.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:12:28 smithi174 ceph-mon[53014]: from='client.34654 -' entity='client.admin' cmd=[{"prefix": "nfs cluster ls", "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:12:30.800 INFO:teuthology.orchestra.run.smithi063.stdout:[ 2024-03-26T21:12:30.801 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-03-26T21:12:30.801 INFO:teuthology.orchestra.run.smithi063.stdout: "access_type": "RW", 2024-03-26T21:12:30.801 INFO:teuthology.orchestra.run.smithi063.stdout: "clients": [], 2024-03-26T21:12:30.801 INFO:teuthology.orchestra.run.smithi063.stdout: "cluster_id": "foo", 2024-03-26T21:12:30.801 INFO:teuthology.orchestra.run.smithi063.stdout: "export_id": 1, 2024-03-26T21:12:30.801 INFO:teuthology.orchestra.run.smithi063.stdout: "fsal": { 2024-03-26T21:12:30.801 INFO:teuthology.orchestra.run.smithi063.stdout: "fs_name": "foofs", 2024-03-26T21:12:30.801 INFO:teuthology.orchestra.run.smithi063.stdout: "name": "CEPH", 2024-03-26T21:12:30.801 INFO:teuthology.orchestra.run.smithi063.stdout: "user_id": "nfs.foo.1" 2024-03-26T21:12:30.801 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-03-26T21:12:30.801 INFO:teuthology.orchestra.run.smithi063.stdout: "path": "/", 2024-03-26T21:12:30.801 INFO:teuthology.orchestra.run.smithi063.stdout: "protocols": [ 2024-03-26T21:12:30.801 INFO:teuthology.orchestra.run.smithi063.stdout: 4 2024-03-26T21:12:30.801 INFO:teuthology.orchestra.run.smithi063.stdout: ], 2024-03-26T21:12:30.801 INFO:teuthology.orchestra.run.smithi063.stdout: "pseudo": "/fake", 2024-03-26T21:12:30.801 INFO:teuthology.orchestra.run.smithi063.stdout: "security_label": true, 2024-03-26T21:12:30.802 INFO:teuthology.orchestra.run.smithi063.stdout: "squash": "none", 2024-03-26T21:12:30.802 INFO:teuthology.orchestra.run.smithi063.stdout: "transports": [ 2024-03-26T21:12:30.802 INFO:teuthology.orchestra.run.smithi063.stdout: "TCP" 2024-03-26T21:12:30.802 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-03-26T21:12:30.802 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-03-26T21:12:30.802 INFO:teuthology.orchestra.run.smithi063.stdout:] 2024-03-26T21:12:31.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:12:30 smithi063 ceph-mon[66934]: pgmap v441: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:12:31.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:12:30 smithi174 ceph-mon[53014]: pgmap v441: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:12:31.479 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -- bash -c 'rados -p .nfs --all ls -' 2024-03-26T21:12:32.051 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:12:31 smithi063 ceph-mon[66934]: from='client.34658 -' entity='client.admin' cmd=[{"prefix": "nfs export ls", "cluster_id": "foo", "detailed": true, "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:12:32.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:12:31 smithi174 ceph-mon[53014]: from='client.34658 -' entity='client.admin' cmd=[{"prefix": "nfs export ls", "cluster_id": "foo", "detailed": true, "target": ["mon-mgr", ""]}]: dispatch 2024-03-26T21:12:33.075 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:12:32 smithi063 ceph-mon[66934]: pgmap v442: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-26T21:12:33.199 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:12:32 smithi174 ceph-mon[53014]: pgmap v442: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-03-26T21:12:33.273 INFO:teuthology.orchestra.run.smithi063.stdout:foo rec-0000000000000004:nfs.foo.1 2024-03-26T21:12:33.273 INFO:teuthology.orchestra.run.smithi063.stdout:foo grace 2024-03-26T21:12:33.273 INFO:teuthology.orchestra.run.smithi063.stdout:foo rec-0000000000000004:nfs.foo.0 2024-03-26T21:12:33.273 INFO:teuthology.orchestra.run.smithi063.stdout:foo conf-nfs.foo 2024-03-26T21:12:33.273 INFO:teuthology.orchestra.run.smithi063.stdout:foo export-1 2024-03-26T21:12:33.858 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-03-26T21:12:33.868 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi063.front.sepia.ceph.com 2024-03-26T21:12:33.868 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 -- bash -c 'set -ex 2024-03-26T21:12:33.868 DEBUG:teuthology.orchestra.run.smithi063:> [[ `ceph config get mgr mgr/cephadm/migration_current` -gt 2 ]] 2024-03-26T21:12:33.868 DEBUG:teuthology.orchestra.run.smithi063:> ' 2024-03-26T21:12:34.185 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:12:33 smithi063 ceph-mon[66934]: mgrmap e43: smithi063.lxhujt(active, since 11m), standbys: smithi174.sexoin 2024-03-26T21:12:34.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:12:33 smithi174 ceph-mon[53014]: mgrmap e43: smithi063.lxhujt(active, since 11m), standbys: smithi174.sexoin 2024-03-26T21:12:35.166 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:12:34 smithi063 ceph-mon[66934]: pgmap v443: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:12:35.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:12:34 smithi174 ceph-mon[53014]: pgmap v443: 97 pgs: 97 active+clean; 225 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-03-26T21:12:35.636 INFO:teuthology.orchestra.run.smithi063.stderr:++ ceph config get mgr mgr/cephadm/migration_current 2024-03-26T21:12:35.880 INFO:teuthology.orchestra.run.smithi063.stderr:+ [[ 6 -gt 2 ]] 2024-03-26T21:12:36.165 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:12:35 smithi063 ceph-mon[66934]: from='client.? 172.21.15.63:0/3468228703' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/cephadm/migration_current"}]: dispatch 2024-03-26T21:12:36.298 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:12:35 smithi174 ceph-mon[53014]: from='client.? 172.21.15.63:0/3468228703' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/cephadm/migration_current"}]: dispatch 2024-03-26T21:12:36.456 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-03-26T21:12:36.466 INFO:tasks.cephadm:Teardown begin 2024-03-26T21:12:36.467 DEBUG:teuthology.orchestra.run.smithi063:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-03-26T21:12:36.494 DEBUG:teuthology.orchestra.run.smithi174:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-03-26T21:12:36.530 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-03-26T21:12:36.530 DEBUG:teuthology.orchestra.run.smithi063:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-03-26T21:12:36.549 DEBUG:teuthology.orchestra.run.smithi174:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-03-26T21:12:36.587 INFO:tasks.cephadm:Stopping all daemons... 2024-03-26T21:12:36.587 INFO:tasks.cephadm.mon.smithi063:Stopping mon.smithi063... 2024-03-26T21:12:36.588 DEBUG:teuthology.orchestra.run.smithi063:> sudo systemctl stop ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715@mon.smithi063 2024-03-26T21:12:36.801 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:12:36 smithi063 systemd[1]: Stopping Ceph mon.smithi063 for 0ed4dd4c-ebb2-11ee-95d0-87774f69a715... 2024-03-26T21:12:37.253 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:12:36 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-mon-smithi063[66930]: 2024-03-26T21:12:36.890+0000 7f97d48c8700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi063 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-03-26T21:12:37.253 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:12:36 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-mon-smithi063[66930]: 2024-03-26T21:12:36.890+0000 7f97d48c8700 -1 mon.smithi063@0(leader) e3 *** Got Signal Terminated *** 2024-03-26T21:12:37.253 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:12:36 smithi063 podman[111979]: 2024-03-26 21:12:36.983377276 +0000 UTC m=+0.194915785 container died c644200344f836ef678909b7171f8409488ec090885f3b7c92402d6254342e25 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-mon-smithi063, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240326, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-03-26T21:12:37.551 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:12:37 smithi063 podman[111979]: 2024-03-26 21:12:37.3671022 +0000 UTC m=+0.578640698 container cleanup c644200344f836ef678909b7171f8409488ec090885f3b7c92402d6254342e25 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-mon-smithi063, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, org.label-schema.build-date=20240326) 2024-03-26T21:12:37.551 INFO:journalctl@ceph.mon.smithi063.smithi063.stdout:Mar 26 21:12:37 smithi063 bash[111979]: ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-mon-smithi063 2024-03-26T21:12:37.818 DEBUG:teuthology.orchestra.run.smithi063:> sudo pkill -f 'journalctl -f -n 0 -u ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715@mon.smithi063.service' 2024-03-26T21:12:37.874 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-26T21:12:37.874 INFO:tasks.cephadm.mon.smithi063:Stopped mon.smithi063 2024-03-26T21:12:37.874 INFO:tasks.cephadm.mon.smithi174:Stopping mon.smithi174... 2024-03-26T21:12:37.874 DEBUG:teuthology.orchestra.run.smithi174:> sudo systemctl stop ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715@mon.smithi174 2024-03-26T21:12:38.226 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:12:37 smithi174 systemd[1]: Stopping Ceph mon.smithi174 for 0ed4dd4c-ebb2-11ee-95d0-87774f69a715... 2024-03-26T21:12:38.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:12:38 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-mon-smithi174[53010]: 2024-03-26T21:12:38.231+0000 7f624b1b8700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi174 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-03-26T21:12:38.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:12:38 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-mon-smithi174[53010]: 2024-03-26T21:12:38.231+0000 7f624b1b8700 -1 mon.smithi174@1(peon) e3 *** Got Signal Terminated *** 2024-03-26T21:12:38.548 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:12:38 smithi174 podman[78726]: 2024-03-26 21:12:38.313548584 +0000 UTC m=+0.234517216 container died 1a6d442692e789ee427d8d48fa89d6faa44467ae591f6c2107e29d458209b5ae (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-mon-smithi174, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0) 2024-03-26T21:12:39.034 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:12:38 smithi174 podman[78726]: 2024-03-26 21:12:38.773148348 +0000 UTC m=+0.694116992 container cleanup 1a6d442692e789ee427d8d48fa89d6faa44467ae591f6c2107e29d458209b5ae (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-mon-smithi174, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS) 2024-03-26T21:12:39.034 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Mar 26 21:12:38 smithi174 bash[78726]: ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-mon-smithi174 2024-03-26T21:12:39.135 DEBUG:teuthology.orchestra.run.smithi174:> sudo pkill -f 'journalctl -f -n 0 -u ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715@mon.smithi174.service' 2024-03-26T21:12:39.178 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-26T21:12:39.178 INFO:tasks.cephadm.mon.smithi174:Stopped mon.smithi174 2024-03-26T21:12:39.178 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-03-26T21:12:39.179 DEBUG:teuthology.orchestra.run.smithi063:> sudo systemctl stop ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715@osd.0 2024-03-26T21:12:39.481 INFO:journalctl@ceph.osd.0.smithi063.stdout:Mar 26 21:12:39 smithi063 systemd[1]: Stopping Ceph osd.0 for 0ed4dd4c-ebb2-11ee-95d0-87774f69a715... 2024-03-26T21:12:39.801 INFO:journalctl@ceph.osd.0.smithi063.stdout:Mar 26 21:12:39 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-0[73102]: 2024-03-26T21:12:39.486+0000 7f4218337700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-26T21:12:39.801 INFO:journalctl@ceph.osd.0.smithi063.stdout:Mar 26 21:12:39 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-0[73102]: 2024-03-26T21:12:39.486+0000 7f4218337700 -1 osd.0 157 *** Got signal Terminated *** 2024-03-26T21:12:39.801 INFO:journalctl@ceph.osd.0.smithi063.stdout:Mar 26 21:12:39 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-0[73102]: 2024-03-26T21:12:39.486+0000 7f4218337700 -1 osd.0 157 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-26T21:12:45.278 INFO:journalctl@ceph.osd.0.smithi063.stdout:Mar 26 21:12:44 smithi063 podman[112111]: 2024-03-26 21:12:44.955114516 +0000 UTC m=+5.587208632 container died 2d4b3fa825cf4317bccdaaad523d19f04ef839b6ee325cb0068c769c15fe11da (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, GIT_CLEAN=True, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=) 2024-03-26T21:12:45.551 INFO:journalctl@ceph.osd.0.smithi063.stdout:Mar 26 21:12:45 smithi063 podman[112111]: 2024-03-26 21:12:45.363100354 +0000 UTC m=+5.995194461 container cleanup 2d4b3fa825cf4317bccdaaad523d19f04ef839b6ee325cb0068c769c15fe11da (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240326, io.buildah.version=1.33.5) 2024-03-26T21:12:45.551 INFO:journalctl@ceph.osd.0.smithi063.stdout:Mar 26 21:12:45 smithi063 bash[112111]: ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-0 2024-03-26T21:12:45.917 INFO:journalctl@ceph.osd.0.smithi063.stdout:Mar 26 21:12:45 smithi063 podman[112160]: 2024-03-26 21:12:45.606120497 +0000 UTC m=+0.650455719 container remove 2d4b3fa825cf4317bccdaaad523d19f04ef839b6ee325cb0068c769c15fe11da (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240326, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, ceph=True) 2024-03-26T21:12:46.228 INFO:journalctl@ceph.osd.0.smithi063.stdout:Mar 26 21:12:45 smithi063 podman[112250]: 2024-03-26 21:12:45.827418417 +0000 UTC m=+0.018528559 image pull a9bdc3eaeb106993bcbbac35d7fe5c7b76c052021676e129f5984aed30aee063 quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d 2024-03-26T21:12:46.228 INFO:journalctl@ceph.osd.0.smithi063.stdout:Mar 26 21:12:45 smithi063 podman[112250]: 2024-03-26 21:12:45.96544595 +0000 UTC m=+0.156556082 container create d595a001aa483d41c319af89ed2f7282938ae283bd67220bea876aa552e1a5c0 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-0-deactivate, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True) 2024-03-26T21:12:46.536 INFO:journalctl@ceph.osd.0.smithi063.stdout:Mar 26 21:12:46 smithi063 podman[112250]: 2024-03-26 21:12:46.234828224 +0000 UTC m=+0.425938365 container init d595a001aa483d41c319af89ed2f7282938ae283bd67220bea876aa552e1a5c0 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-0-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240326, ceph=True, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b) 2024-03-26T21:12:46.536 INFO:journalctl@ceph.osd.0.smithi063.stdout:Mar 26 21:12:46 smithi063 podman[112250]: 2024-03-26 21:12:46.239104818 +0000 UTC m=+0.430214953 container start d595a001aa483d41c319af89ed2f7282938ae283bd67220bea876aa552e1a5c0 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-0-deactivate, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, org.label-schema.build-date=20240326, ceph=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-03-26T21:12:46.537 INFO:journalctl@ceph.osd.0.smithi063.stdout:Mar 26 21:12:46 smithi063 podman[112250]: 2024-03-26 21:12:46.309609043 +0000 UTC m=+0.500719194 container attach d595a001aa483d41c319af89ed2f7282938ae283bd67220bea876aa552e1a5c0 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-0-deactivate, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.33.5, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240326) 2024-03-26T21:12:46.801 INFO:journalctl@ceph.osd.0.smithi063.stdout:Mar 26 21:12:46 smithi063 podman[112250]: 2024-03-26 21:12:46.543933416 +0000 UTC m=+0.735043549 container died d595a001aa483d41c319af89ed2f7282938ae283bd67220bea876aa552e1a5c0 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-0-deactivate, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, org.label-schema.license=GPLv2, org.label-schema.build-date=20240326, io.buildah.version=1.33.5, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-26T21:12:47.229 DEBUG:teuthology.orchestra.run.smithi063:> sudo pkill -f 'journalctl -f -n 0 -u ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715@osd.0.service' 2024-03-26T21:12:47.269 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-26T21:12:47.269 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-03-26T21:12:47.269 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-03-26T21:12:47.269 DEBUG:teuthology.orchestra.run.smithi063:> sudo systemctl stop ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715@osd.1 2024-03-26T21:12:47.551 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 21:12:47 smithi063 systemd[1]: Stopping Ceph osd.1 for 0ed4dd4c-ebb2-11ee-95d0-87774f69a715... 2024-03-26T21:12:48.051 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 21:12:47 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-1[75620]: 2024-03-26T21:12:47.632+0000 7f39876e9700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-26T21:12:48.051 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 21:12:47 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-1[75620]: 2024-03-26T21:12:47.632+0000 7f39876e9700 -1 osd.1 157 *** Got signal Terminated *** 2024-03-26T21:12:48.051 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 21:12:47 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-1[75620]: 2024-03-26T21:12:47.632+0000 7f39876e9700 -1 osd.1 157 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-26T21:12:53.318 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 21:12:53 smithi063 podman[112356]: 2024-03-26 21:12:53.061137439 +0000 UTC m=+5.550126485 container died d41fbe507197d662b5c28b38a0faee89b3db907f4774e9f34c8dc10b6b6a77e7 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-1, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-26T21:12:53.593 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 21:12:53 smithi063 podman[112356]: 2024-03-26 21:12:53.399081268 +0000 UTC m=+5.888070313 container cleanup d41fbe507197d662b5c28b38a0faee89b3db907f4774e9f34c8dc10b6b6a77e7 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-1, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=) 2024-03-26T21:12:53.593 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 21:12:53 smithi063 bash[112356]: ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-1 2024-03-26T21:12:54.044 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 21:12:53 smithi063 podman[112376]: 2024-03-26 21:12:53.600467448 +0000 UTC m=+0.536074536 container remove d41fbe507197d662b5c28b38a0faee89b3db907f4774e9f34c8dc10b6b6a77e7 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-1, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240326, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.33.5, ceph=True) 2024-03-26T21:12:54.301 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 21:12:54 smithi063 podman[112467]: 2024-03-26 21:12:53.954891005 +0000 UTC m=+0.020218459 image pull a9bdc3eaeb106993bcbbac35d7fe5c7b76c052021676e129f5984aed30aee063 quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d 2024-03-26T21:12:54.301 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 21:12:54 smithi063 podman[112467]: 2024-03-26 21:12:54.101326075 +0000 UTC m=+0.166653512 container create 8abef80280562eca31ccb7e7a7f235d5ea01e4116ebc263ada91d3e9cc954afd (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-1-deactivate, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, ceph=True) 2024-03-26T21:12:54.694 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 21:12:54 smithi063 podman[112467]: 2024-03-26 21:12:54.364504256 +0000 UTC m=+0.429831693 container init 8abef80280562eca31ccb7e7a7f235d5ea01e4116ebc263ada91d3e9cc954afd (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-1-deactivate, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, org.label-schema.license=GPLv2, ceph=True) 2024-03-26T21:12:54.694 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 21:12:54 smithi063 podman[112467]: 2024-03-26 21:12:54.367821051 +0000 UTC m=+0.433148495 container start 8abef80280562eca31ccb7e7a7f235d5ea01e4116ebc263ada91d3e9cc954afd (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-1-deactivate, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-03-26T21:12:54.694 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 21:12:54 smithi063 podman[112467]: 2024-03-26 21:12:54.44790436 +0000 UTC m=+0.513231820 container attach 8abef80280562eca31ccb7e7a7f235d5ea01e4116ebc263ada91d3e9cc954afd (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-1-deactivate, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, org.label-schema.schema-version=1.0) 2024-03-26T21:12:55.002 INFO:journalctl@ceph.osd.1.smithi063.stdout:Mar 26 21:12:54 smithi063 podman[112467]: 2024-03-26 21:12:54.701468334 +0000 UTC m=+0.766795781 container died 8abef80280562eca31ccb7e7a7f235d5ea01e4116ebc263ada91d3e9cc954afd (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-1-deactivate, GIT_CLEAN=True, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-26T21:12:55.305 DEBUG:teuthology.orchestra.run.smithi063:> sudo pkill -f 'journalctl -f -n 0 -u ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715@osd.1.service' 2024-03-26T21:12:55.346 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-26T21:12:55.346 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-03-26T21:12:55.346 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-03-26T21:12:55.347 DEBUG:teuthology.orchestra.run.smithi063:> sudo systemctl stop ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715@osd.2 2024-03-26T21:12:55.551 INFO:journalctl@ceph.osd.2.smithi063.stdout:Mar 26 21:12:55 smithi063 systemd[1]: Stopping Ceph osd.2 for 0ed4dd4c-ebb2-11ee-95d0-87774f69a715... 2024-03-26T21:12:56.051 INFO:journalctl@ceph.osd.2.smithi063.stdout:Mar 26 21:12:55 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-2[78244]: 2024-03-26T21:12:55.712+0000 7fa9bf909700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-26T21:12:56.051 INFO:journalctl@ceph.osd.2.smithi063.stdout:Mar 26 21:12:55 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-2[78244]: 2024-03-26T21:12:55.712+0000 7fa9bf909700 -1 osd.2 157 *** Got signal Terminated *** 2024-03-26T21:12:56.051 INFO:journalctl@ceph.osd.2.smithi063.stdout:Mar 26 21:12:55 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-2[78244]: 2024-03-26T21:12:55.712+0000 7fa9bf909700 -1 osd.2 157 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-26T21:13:01.524 INFO:journalctl@ceph.osd.2.smithi063.stdout:Mar 26 21:13:01 smithi063 podman[112572]: 2024-03-26 21:13:01.231291386 +0000 UTC m=+5.642036315 container died f123eeabd3fa7d5076b1f7427d92ae5367f7aea4ec0eb6aa34678d6417edafaf (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, GIT_CLEAN=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.5) 2024-03-26T21:13:01.801 INFO:journalctl@ceph.osd.2.smithi063.stdout:Mar 26 21:13:01 smithi063 podman[112572]: 2024-03-26 21:13:01.629976192 +0000 UTC m=+6.040721112 container cleanup f123eeabd3fa7d5076b1f7427d92ae5367f7aea4ec0eb6aa34678d6417edafaf (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, ceph=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-03-26T21:13:01.801 INFO:journalctl@ceph.osd.2.smithi063.stdout:Mar 26 21:13:01 smithi063 bash[112572]: ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-2 2024-03-26T21:13:02.201 INFO:journalctl@ceph.osd.2.smithi063.stdout:Mar 26 21:13:01 smithi063 podman[112623]: 2024-03-26 21:13:01.864605413 +0000 UTC m=+0.635465722 container remove f123eeabd3fa7d5076b1f7427d92ae5367f7aea4ec0eb6aa34678d6417edafaf (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-2, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux ) 2024-03-26T21:13:02.520 INFO:journalctl@ceph.osd.2.smithi063.stdout:Mar 26 21:13:02 smithi063 podman[112713]: 2024-03-26 21:13:02.111624481 +0000 UTC m=+0.018432485 image pull a9bdc3eaeb106993bcbbac35d7fe5c7b76c052021676e129f5984aed30aee063 quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d 2024-03-26T21:13:02.520 INFO:journalctl@ceph.osd.2.smithi063.stdout:Mar 26 21:13:02 smithi063 podman[112713]: 2024-03-26 21:13:02.232207272 +0000 UTC m=+0.139015259 container create 2e8d0442bc3c13f28a2c639162ed7b75937b039f3cbd036471b730cc52635974 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-2-deactivate, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, GIT_CLEAN=True, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, ceph=True, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=) 2024-03-26T21:13:02.520 INFO:journalctl@ceph.osd.2.smithi063.stdout:Mar 26 21:13:02 smithi063 podman[112713]: 2024-03-26 21:13:02.427215496 +0000 UTC m=+0.334023479 container init 2e8d0442bc3c13f28a2c639162ed7b75937b039f3cbd036471b730cc52635974 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-2-deactivate, CEPH_POINT_RELEASE=, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True) 2024-03-26T21:13:02.520 INFO:journalctl@ceph.osd.2.smithi063.stdout:Mar 26 21:13:02 smithi063 podman[112713]: 2024-03-26 21:13:02.43046472 +0000 UTC m=+0.337272707 container start 2e8d0442bc3c13f28a2c639162ed7b75937b039f3cbd036471b730cc52635974 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-2-deactivate, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240326, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-26T21:13:02.520 INFO:journalctl@ceph.osd.2.smithi063.stdout:Mar 26 21:13:02 smithi063 podman[112713]: 2024-03-26 21:13:02.526462122 +0000 UTC m=+0.433270110 container attach 2e8d0442bc3c13f28a2c639162ed7b75937b039f3cbd036471b730cc52635974 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-2-deactivate, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.build-date=20240326, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-03-26T21:13:02.801 INFO:journalctl@ceph.osd.2.smithi063.stdout:Mar 26 21:13:02 smithi063 podman[112713]: 2024-03-26 21:13:02.724005029 +0000 UTC m=+0.630813024 container died 2e8d0442bc3c13f28a2c639162ed7b75937b039f3cbd036471b730cc52635974 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-2-deactivate, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-03-26T21:13:03.517 DEBUG:teuthology.orchestra.run.smithi063:> sudo pkill -f 'journalctl -f -n 0 -u ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715@osd.2.service' 2024-03-26T21:13:03.557 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-26T21:13:03.557 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-03-26T21:13:03.557 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-03-26T21:13:03.557 DEBUG:teuthology.orchestra.run.smithi063:> sudo systemctl stop ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715@osd.3 2024-03-26T21:13:03.801 INFO:journalctl@ceph.osd.3.smithi063.stdout:Mar 26 21:13:03 smithi063 systemd[1]: Stopping Ceph osd.3 for 0ed4dd4c-ebb2-11ee-95d0-87774f69a715... 2024-03-26T21:13:04.301 INFO:journalctl@ceph.osd.3.smithi063.stdout:Mar 26 21:13:03 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-3[80818]: 2024-03-26T21:13:03.898+0000 7f3c5d400700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-26T21:13:04.301 INFO:journalctl@ceph.osd.3.smithi063.stdout:Mar 26 21:13:03 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-3[80818]: 2024-03-26T21:13:03.898+0000 7f3c5d400700 -1 osd.3 157 *** Got signal Terminated *** 2024-03-26T21:13:04.301 INFO:journalctl@ceph.osd.3.smithi063.stdout:Mar 26 21:13:03 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-3[80818]: 2024-03-26T21:13:03.898+0000 7f3c5d400700 -1 osd.3 157 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-26T21:13:06.548 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:13:06 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6[62438]: 2024-03-26T21:13:06.192+0000 7f05ca911700 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:43.011095+0000 front 2024-03-26T21:12:43.010968+0000 (oldest deadline 2024-03-26T21:13:05.911029+0000) 2024-03-26T21:13:07.051 INFO:journalctl@ceph.osd.3.smithi063.stdout:Mar 26 21:13:06 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-3[80818]: 2024-03-26T21:13:06.628+0000 7f3c559ec700 -1 osd.3 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:42.738161+0000 front 2024-03-26T21:12:42.738178+0000 (oldest deadline 2024-03-26T21:13:06.238298+0000) 2024-03-26T21:13:07.157 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:13:06 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-5[60157]: 2024-03-26T21:13:06.889+0000 7f2813359700 -1 osd.5 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:41.335725+0000 front 2024-03-26T21:12:41.335743+0000 (oldest deadline 2024-03-26T21:13:06.035916+0000) 2024-03-26T21:13:07.548 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:13:07 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6[62438]: 2024-03-26T21:13:07.163+0000 7f05ca911700 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:43.011095+0000 front 2024-03-26T21:12:43.010968+0000 (oldest deadline 2024-03-26T21:13:05.911029+0000) 2024-03-26T21:13:08.048 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:13:07 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-5[60157]: 2024-03-26T21:13:07.935+0000 7f2813359700 -1 osd.5 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:41.335725+0000 front 2024-03-26T21:12:41.335743+0000 (oldest deadline 2024-03-26T21:13:06.035916+0000) 2024-03-26T21:13:08.051 INFO:journalctl@ceph.osd.3.smithi063.stdout:Mar 26 21:13:07 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-3[80818]: 2024-03-26T21:13:07.579+0000 7f3c559ec700 -1 osd.3 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:42.738161+0000 front 2024-03-26T21:12:42.738178+0000 (oldest deadline 2024-03-26T21:13:06.238298+0000) 2024-03-26T21:13:08.548 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:13:08 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6[62438]: 2024-03-26T21:13:08.149+0000 7f05ca911700 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:43.011095+0000 front 2024-03-26T21:12:43.010968+0000 (oldest deadline 2024-03-26T21:13:05.911029+0000) 2024-03-26T21:13:08.548 INFO:journalctl@ceph.osd.4.smithi174.stdout:Mar 26 21:13:08 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-4[57895]: 2024-03-26T21:13:08.397+0000 7f88af542700 -1 osd.4 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:42.944096+0000 front 2024-03-26T21:12:42.944155+0000 (oldest deadline 2024-03-26T21:13:08.244091+0000) 2024-03-26T21:13:09.051 INFO:journalctl@ceph.osd.3.smithi063.stdout:Mar 26 21:13:08 smithi063 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-3[80818]: 2024-03-26T21:13:08.613+0000 7f3c559ec700 -1 osd.3 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:42.738161+0000 front 2024-03-26T21:12:42.738178+0000 (oldest deadline 2024-03-26T21:13:06.238298+0000) 2024-03-26T21:13:09.298 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:13:09 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6[62438]: 2024-03-26T21:13:09.121+0000 7f05ca911700 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:43.011095+0000 front 2024-03-26T21:12:43.010968+0000 (oldest deadline 2024-03-26T21:13:05.911029+0000) 2024-03-26T21:13:09.298 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:13:08 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-5[60157]: 2024-03-26T21:13:08.961+0000 7f2813359700 -1 osd.5 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:41.335725+0000 front 2024-03-26T21:12:41.335743+0000 (oldest deadline 2024-03-26T21:13:06.035916+0000) 2024-03-26T21:13:09.752 INFO:journalctl@ceph.osd.3.smithi063.stdout:Mar 26 21:13:09 smithi063 podman[112817]: 2024-03-26 21:13:09.477907273 +0000 UTC m=+5.680557455 container died 953816eafe87e6dacd376c5e8ed51d8307d0b077830d4573c89303a7f2f59183 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-3, io.buildah.version=1.33.5, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e) 2024-03-26T21:13:09.798 INFO:journalctl@ceph.osd.4.smithi174.stdout:Mar 26 21:13:09 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-4[57895]: 2024-03-26T21:13:09.409+0000 7f88af542700 -1 osd.4 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:42.944096+0000 front 2024-03-26T21:12:42.944155+0000 (oldest deadline 2024-03-26T21:13:08.244091+0000) 2024-03-26T21:13:09.798 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:09 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:09.348+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:41.080008+0000 front 2024-03-26T21:12:41.080165+0000 (oldest deadline 2024-03-26T21:13:08.680479+0000) 2024-03-26T21:13:10.051 INFO:journalctl@ceph.osd.3.smithi063.stdout:Mar 26 21:13:09 smithi063 podman[112817]: 2024-03-26 21:13:09.832856298 +0000 UTC m=+6.035506480 container cleanup 953816eafe87e6dacd376c5e8ed51d8307d0b077830d4573c89303a7f2f59183 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-3, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2) 2024-03-26T21:13:10.051 INFO:journalctl@ceph.osd.3.smithi063.stdout:Mar 26 21:13:09 smithi063 bash[112817]: ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-3 2024-03-26T21:13:10.298 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:13:10 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6[62438]: 2024-03-26T21:13:10.093+0000 7f05ca911700 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:43.011095+0000 front 2024-03-26T21:12:43.010968+0000 (oldest deadline 2024-03-26T21:13:05.911029+0000) 2024-03-26T21:13:10.298 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:13:09 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-5[60157]: 2024-03-26T21:13:09.915+0000 7f2813359700 -1 osd.5 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:41.335725+0000 front 2024-03-26T21:12:41.335743+0000 (oldest deadline 2024-03-26T21:13:06.035916+0000) 2024-03-26T21:13:10.449 INFO:journalctl@ceph.osd.3.smithi063.stdout:Mar 26 21:13:10 smithi063 podman[112834]: 2024-03-26 21:13:10.10074105 +0000 UTC m=+0.626520955 container remove 953816eafe87e6dacd376c5e8ed51d8307d0b077830d4573c89303a7f2f59183 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-3, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240326, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.33.5) 2024-03-26T21:13:10.714 INFO:journalctl@ceph.osd.3.smithi063.stdout:Mar 26 21:13:10 smithi063 podman[112924]: 2024-03-26 21:13:10.361847306 +0000 UTC m=+0.020712015 image pull a9bdc3eaeb106993bcbbac35d7fe5c7b76c052021676e129f5984aed30aee063 quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d 2024-03-26T21:13:10.714 INFO:journalctl@ceph.osd.3.smithi063.stdout:Mar 26 21:13:10 smithi063 podman[112924]: 2024-03-26 21:13:10.47666291 +0000 UTC m=+0.135527603 container create d99e604759515dd37797f4eb0a69603c210a442bb810a4b4af5c077f9f9db153 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-3-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b) 2024-03-26T21:13:10.714 INFO:journalctl@ceph.osd.3.smithi063.stdout:Mar 26 21:13:10 smithi063 podman[112924]: 2024-03-26 21:13:10.721220377 +0000 UTC m=+0.380085072 container init d99e604759515dd37797f4eb0a69603c210a442bb810a4b4af5c077f9f9db153 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-3-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326, io.buildah.version=1.33.5, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, GIT_CLEAN=True) 2024-03-26T21:13:10.798 INFO:journalctl@ceph.osd.4.smithi174.stdout:Mar 26 21:13:10 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-4[57895]: 2024-03-26T21:13:10.412+0000 7f88af542700 -1 osd.4 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:42.944096+0000 front 2024-03-26T21:12:42.944155+0000 (oldest deadline 2024-03-26T21:13:08.244091+0000) 2024-03-26T21:13:10.798 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:10 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:10.350+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:41.080008+0000 front 2024-03-26T21:12:41.080165+0000 (oldest deadline 2024-03-26T21:13:08.680479+0000) 2024-03-26T21:13:11.025 INFO:journalctl@ceph.osd.3.smithi063.stdout:Mar 26 21:13:10 smithi063 podman[112924]: 2024-03-26 21:13:10.724576901 +0000 UTC m=+0.383441598 container start d99e604759515dd37797f4eb0a69603c210a442bb810a4b4af5c077f9f9db153 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-3-deactivate, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b) 2024-03-26T21:13:11.025 INFO:journalctl@ceph.osd.3.smithi063.stdout:Mar 26 21:13:10 smithi063 podman[112924]: 2024-03-26 21:13:10.854591955 +0000 UTC m=+0.513456667 container attach d99e604759515dd37797f4eb0a69603c210a442bb810a4b4af5c077f9f9db153 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-3-deactivate, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-26T21:13:11.298 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:13:10 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-5[60157]: 2024-03-26T21:13:10.871+0000 7f2813359700 -1 osd.5 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:41.335725+0000 front 2024-03-26T21:12:41.335743+0000 (oldest deadline 2024-03-26T21:13:06.035916+0000) 2024-03-26T21:13:11.298 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:13:11 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6[62438]: 2024-03-26T21:13:11.115+0000 7f05ca911700 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:43.011095+0000 front 2024-03-26T21:12:43.010968+0000 (oldest deadline 2024-03-26T21:13:05.911029+0000) 2024-03-26T21:13:11.301 INFO:journalctl@ceph.osd.3.smithi063.stdout:Mar 26 21:13:11 smithi063 podman[112924]: 2024-03-26 21:13:11.032096199 +0000 UTC m=+0.690960897 container died d99e604759515dd37797f4eb0a69603c210a442bb810a4b4af5c077f9f9db153 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-3-deactivate, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, org.label-schema.build-date=20240326, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0) 2024-03-26T21:13:11.565 INFO:journalctl@ceph.osd.3.smithi063.stdout:Mar 26 21:13:11 smithi063 podman[112950]: 2024-03-26 21:13:11.55788923 +0000 UTC m=+0.520878919 container remove d99e604759515dd37797f4eb0a69603c210a442bb810a4b4af5c077f9f9db153 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-3-deactivate, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, org.label-schema.build-date=20240326, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-26T21:13:11.570 DEBUG:teuthology.orchestra.run.smithi063:> sudo pkill -f 'journalctl -f -n 0 -u ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715@osd.3.service' 2024-03-26T21:13:11.612 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-26T21:13:11.613 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-03-26T21:13:11.613 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-03-26T21:13:11.613 DEBUG:teuthology.orchestra.run.smithi174:> sudo systemctl stop ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715@osd.4 2024-03-26T21:13:11.644 INFO:journalctl@ceph.osd.4.smithi174.stdout:Mar 26 21:13:11 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-4[57895]: 2024-03-26T21:13:11.389+0000 7f88af542700 -1 osd.4 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:42.944096+0000 front 2024-03-26T21:12:42.944155+0000 (oldest deadline 2024-03-26T21:13:08.244091+0000) 2024-03-26T21:13:11.644 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:11 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:11.320+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:41.080008+0000 front 2024-03-26T21:12:41.080165+0000 (oldest deadline 2024-03-26T21:13:08.680479+0000) 2024-03-26T21:13:11.952 INFO:journalctl@ceph.osd.4.smithi174.stdout:Mar 26 21:13:11 smithi174 systemd[1]: Stopping Ceph osd.4 for 0ed4dd4c-ebb2-11ee-95d0-87774f69a715... 2024-03-26T21:13:11.952 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:13:11 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-5[60157]: 2024-03-26T21:13:11.860+0000 7f2813359700 -1 osd.5 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:41.335725+0000 front 2024-03-26T21:12:41.335743+0000 (oldest deadline 2024-03-26T21:13:06.035916+0000) 2024-03-26T21:13:12.270 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:13:12 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6[62438]: 2024-03-26T21:13:12.121+0000 7f05ca911700 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:43.011095+0000 front 2024-03-26T21:12:43.010968+0000 (oldest deadline 2024-03-26T21:13:05.911029+0000) 2024-03-26T21:13:12.271 INFO:journalctl@ceph.osd.4.smithi174.stdout:Mar 26 21:13:11 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-4[57895]: 2024-03-26T21:13:11.958+0000 7f88b2f4e700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-26T21:13:12.271 INFO:journalctl@ceph.osd.4.smithi174.stdout:Mar 26 21:13:11 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-4[57895]: 2024-03-26T21:13:11.958+0000 7f88b2f4e700 -1 osd.4 157 *** Got signal Terminated *** 2024-03-26T21:13:12.271 INFO:journalctl@ceph.osd.4.smithi174.stdout:Mar 26 21:13:11 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-4[57895]: 2024-03-26T21:13:11.958+0000 7f88b2f4e700 -1 osd.4 157 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-26T21:13:12.548 INFO:journalctl@ceph.osd.4.smithi174.stdout:Mar 26 21:13:12 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-4[57895]: 2024-03-26T21:13:12.364+0000 7f88af542700 -1 osd.4 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:42.944096+0000 front 2024-03-26T21:12:42.944155+0000 (oldest deadline 2024-03-26T21:13:08.244091+0000) 2024-03-26T21:13:12.549 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:12 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:12.276+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:41.080008+0000 front 2024-03-26T21:12:41.080165+0000 (oldest deadline 2024-03-26T21:13:08.680479+0000) 2024-03-26T21:13:13.254 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:13:13 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6[62438]: 2024-03-26T21:13:13.091+0000 7f05ca911700 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:43.011095+0000 front 2024-03-26T21:12:43.010968+0000 (oldest deadline 2024-03-26T21:13:05.911029+0000) 2024-03-26T21:13:13.254 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:13:12 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-5[60157]: 2024-03-26T21:13:12.905+0000 7f2813359700 -1 osd.5 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:41.335725+0000 front 2024-03-26T21:12:41.335743+0000 (oldest deadline 2024-03-26T21:13:06.035916+0000) 2024-03-26T21:13:13.548 INFO:journalctl@ceph.osd.4.smithi174.stdout:Mar 26 21:13:13 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-4[57895]: 2024-03-26T21:13:13.333+0000 7f88af542700 -1 osd.4 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:42.944096+0000 front 2024-03-26T21:12:42.944155+0000 (oldest deadline 2024-03-26T21:13:08.244091+0000) 2024-03-26T21:13:13.548 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:13 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:13.260+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:41.080008+0000 front 2024-03-26T21:12:41.080165+0000 (oldest deadline 2024-03-26T21:13:08.680479+0000) 2024-03-26T21:13:14.229 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:13:13 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-5[60157]: 2024-03-26T21:13:13.945+0000 7f2813359700 -1 osd.5 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:41.335725+0000 front 2024-03-26T21:12:41.335743+0000 (oldest deadline 2024-03-26T21:13:06.035916+0000) 2024-03-26T21:13:14.229 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:13:14 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6[62438]: 2024-03-26T21:13:14.133+0000 7f05ca911700 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:43.011095+0000 front 2024-03-26T21:12:43.010968+0000 (oldest deadline 2024-03-26T21:13:05.911029+0000) 2024-03-26T21:13:14.229 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:13:14 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6[62438]: 2024-03-26T21:13:14.133+0000 7f05ca911700 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-03-26T21:12:47.612134+0000 front 2024-03-26T21:12:47.612161+0000 (oldest deadline 2024-03-26T21:13:13.512031+0000) 2024-03-26T21:13:14.548 INFO:journalctl@ceph.osd.4.smithi174.stdout:Mar 26 21:13:14 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-4[57895]: 2024-03-26T21:13:14.360+0000 7f88af542700 -1 osd.4 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:42.944096+0000 front 2024-03-26T21:12:42.944155+0000 (oldest deadline 2024-03-26T21:13:08.244091+0000) 2024-03-26T21:13:14.548 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:14 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:14.235+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:41.080008+0000 front 2024-03-26T21:12:41.080165+0000 (oldest deadline 2024-03-26T21:13:08.680479+0000) 2024-03-26T21:13:14.548 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:14 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:14.235+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-03-26T21:12:48.680857+0000 front 2024-03-26T21:12:48.680883+0000 (oldest deadline 2024-03-26T21:13:13.981129+0000) 2024-03-26T21:13:15.298 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:13:14 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-5[60157]: 2024-03-26T21:13:14.995+0000 7f2813359700 -1 osd.5 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:41.335725+0000 front 2024-03-26T21:12:41.335743+0000 (oldest deadline 2024-03-26T21:13:06.035916+0000) 2024-03-26T21:13:15.298 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:13:15 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6[62438]: 2024-03-26T21:13:15.127+0000 7f05ca911700 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:43.011095+0000 front 2024-03-26T21:12:43.010968+0000 (oldest deadline 2024-03-26T21:13:05.911029+0000) 2024-03-26T21:13:15.298 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:13:15 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6[62438]: 2024-03-26T21:13:15.127+0000 7f05ca911700 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-03-26T21:12:47.612134+0000 front 2024-03-26T21:12:47.612161+0000 (oldest deadline 2024-03-26T21:13:13.512031+0000) 2024-03-26T21:13:15.299 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:15 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:15.218+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:41.080008+0000 front 2024-03-26T21:12:41.080165+0000 (oldest deadline 2024-03-26T21:13:08.680479+0000) 2024-03-26T21:13:15.299 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:15 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:15.218+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-03-26T21:12:48.680857+0000 front 2024-03-26T21:12:48.680883+0000 (oldest deadline 2024-03-26T21:13:13.981129+0000) 2024-03-26T21:13:15.798 INFO:journalctl@ceph.osd.4.smithi174.stdout:Mar 26 21:13:15 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-4[57895]: 2024-03-26T21:13:15.404+0000 7f88af542700 -1 osd.4 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:42.944096+0000 front 2024-03-26T21:12:42.944155+0000 (oldest deadline 2024-03-26T21:13:08.244091+0000) 2024-03-26T21:13:16.298 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:13:15 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-5[60157]: 2024-03-26T21:13:15.957+0000 7f2813359700 -1 osd.5 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:41.335725+0000 front 2024-03-26T21:12:41.335743+0000 (oldest deadline 2024-03-26T21:13:06.035916+0000) 2024-03-26T21:13:16.298 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:13:16 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6[62438]: 2024-03-26T21:13:16.102+0000 7f05ca911700 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:43.011095+0000 front 2024-03-26T21:12:43.010968+0000 (oldest deadline 2024-03-26T21:13:05.911029+0000) 2024-03-26T21:13:16.298 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:13:16 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6[62438]: 2024-03-26T21:13:16.102+0000 7f05ca911700 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-03-26T21:12:47.612134+0000 front 2024-03-26T21:12:47.612161+0000 (oldest deadline 2024-03-26T21:13:13.512031+0000) 2024-03-26T21:13:16.299 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:16 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:16.175+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:41.080008+0000 front 2024-03-26T21:12:41.080165+0000 (oldest deadline 2024-03-26T21:13:08.680479+0000) 2024-03-26T21:13:16.299 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:16 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:16.175+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-03-26T21:12:48.680857+0000 front 2024-03-26T21:12:48.680883+0000 (oldest deadline 2024-03-26T21:13:13.981129+0000) 2024-03-26T21:13:16.798 INFO:journalctl@ceph.osd.4.smithi174.stdout:Mar 26 21:13:16 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-4[57895]: 2024-03-26T21:13:16.380+0000 7f88af542700 -1 osd.4 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:42.944096+0000 front 2024-03-26T21:12:42.944155+0000 (oldest deadline 2024-03-26T21:13:08.244091+0000) 2024-03-26T21:13:17.298 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:13:16 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-5[60157]: 2024-03-26T21:13:16.966+0000 7f2813359700 -1 osd.5 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:41.335725+0000 front 2024-03-26T21:12:41.335743+0000 (oldest deadline 2024-03-26T21:13:06.035916+0000) 2024-03-26T21:13:17.298 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:13:17 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6[62438]: 2024-03-26T21:13:17.095+0000 7f05ca911700 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:43.011095+0000 front 2024-03-26T21:12:43.010968+0000 (oldest deadline 2024-03-26T21:13:05.911029+0000) 2024-03-26T21:13:17.298 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:13:17 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6[62438]: 2024-03-26T21:13:17.095+0000 7f05ca911700 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-03-26T21:12:47.612134+0000 front 2024-03-26T21:12:47.612161+0000 (oldest deadline 2024-03-26T21:13:13.512031+0000) 2024-03-26T21:13:17.299 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:17 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:17.177+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:41.080008+0000 front 2024-03-26T21:12:41.080165+0000 (oldest deadline 2024-03-26T21:13:08.680479+0000) 2024-03-26T21:13:17.299 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:17 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:17.177+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-03-26T21:12:48.680857+0000 front 2024-03-26T21:12:48.680883+0000 (oldest deadline 2024-03-26T21:13:13.981129+0000) 2024-03-26T21:13:17.687 INFO:journalctl@ceph.osd.4.smithi174.stdout:Mar 26 21:13:17 smithi174 podman[78923]: 2024-03-26 21:13:17.401445702 +0000 UTC m=+5.592243954 container died 8aa35bff6ceace743b9eeec72019e4f30803a59ea5021a63423ea10f8b905084 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-4, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e) 2024-03-26T21:13:17.996 INFO:journalctl@ceph.osd.4.smithi174.stdout:Mar 26 21:13:17 smithi174 podman[78923]: 2024-03-26 21:13:17.876096516 +0000 UTC m=+6.066894830 container cleanup 8aa35bff6ceace743b9eeec72019e4f30803a59ea5021a63423ea10f8b905084 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-4, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, org.label-schema.build-date=20240326, io.buildah.version=1.33.5) 2024-03-26T21:13:17.996 INFO:journalctl@ceph.osd.4.smithi174.stdout:Mar 26 21:13:17 smithi174 bash[78923]: ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-4 2024-03-26T21:13:18.298 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:13:18 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-5[60157]: 2024-03-26T21:13:18.002+0000 7f2813359700 -1 osd.5 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:41.335725+0000 front 2024-03-26T21:12:41.335743+0000 (oldest deadline 2024-03-26T21:13:06.035916+0000) 2024-03-26T21:13:18.298 INFO:journalctl@ceph.osd.4.smithi174.stdout:Mar 26 21:13:18 smithi174 podman[78942]: 2024-03-26 21:13:18.127125303 +0000 UTC m=+0.720792080 container remove 8aa35bff6ceace743b9eeec72019e4f30803a59ea5021a63423ea10f8b905084 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-4, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, ceph=True) 2024-03-26T21:13:18.299 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:13:18 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6[62438]: 2024-03-26T21:13:18.103+0000 7f05ca911700 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:43.011095+0000 front 2024-03-26T21:12:43.010968+0000 (oldest deadline 2024-03-26T21:13:05.911029+0000) 2024-03-26T21:13:18.299 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:13:18 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6[62438]: 2024-03-26T21:13:18.103+0000 7f05ca911700 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-03-26T21:12:47.612134+0000 front 2024-03-26T21:12:47.612161+0000 (oldest deadline 2024-03-26T21:13:13.512031+0000) 2024-03-26T21:13:18.299 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:18 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:18.179+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:41.080008+0000 front 2024-03-26T21:12:41.080165+0000 (oldest deadline 2024-03-26T21:13:08.680479+0000) 2024-03-26T21:13:18.299 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:18 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:18.179+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-03-26T21:12:48.680857+0000 front 2024-03-26T21:12:48.680883+0000 (oldest deadline 2024-03-26T21:13:13.981129+0000) 2024-03-26T21:13:18.782 INFO:journalctl@ceph.osd.4.smithi174.stdout:Mar 26 21:13:18 smithi174 podman[79034]: 2024-03-26 21:13:18.387982126 +0000 UTC m=+0.017140329 image pull a9bdc3eaeb106993bcbbac35d7fe5c7b76c052021676e129f5984aed30aee063 quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d 2024-03-26T21:13:18.782 INFO:journalctl@ceph.osd.4.smithi174.stdout:Mar 26 21:13:18 smithi174 podman[79034]: 2024-03-26 21:13:18.486358262 +0000 UTC m=+0.115516486 container create db957ffc408a0f41014a022753f5265f1e0237c0a5d25456987a3d4514177e24 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-4-deactivate, org.label-schema.build-date=20240326, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, GIT_BRANCH=HEAD) 2024-03-26T21:13:18.782 INFO:journalctl@ceph.osd.4.smithi174.stdout:Mar 26 21:13:18 smithi174 podman[79034]: 2024-03-26 21:13:18.705464173 +0000 UTC m=+0.334622351 container init db957ffc408a0f41014a022753f5265f1e0237c0a5d25456987a3d4514177e24 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-4-deactivate, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True) 2024-03-26T21:13:18.782 INFO:journalctl@ceph.osd.4.smithi174.stdout:Mar 26 21:13:18 smithi174 podman[79034]: 2024-03-26 21:13:18.708597242 +0000 UTC m=+0.337755422 container start db957ffc408a0f41014a022753f5265f1e0237c0a5d25456987a3d4514177e24 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-4-deactivate, CEPH_POINT_RELEASE=, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240326) 2024-03-26T21:13:18.782 INFO:journalctl@ceph.osd.4.smithi174.stdout:Mar 26 21:13:18 smithi174 podman[79034]: 2024-03-26 21:13:18.788597029 +0000 UTC m=+0.417755213 container attach db957ffc408a0f41014a022753f5265f1e0237c0a5d25456987a3d4514177e24 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-4-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240326, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, GIT_BRANCH=HEAD, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux ) 2024-03-26T21:13:19.034 INFO:journalctl@ceph.osd.4.smithi174.stdout:Mar 26 21:13:19 smithi174 podman[79034]: 2024-03-26 21:13:19.004655185 +0000 UTC m=+0.633813375 container died db957ffc408a0f41014a022753f5265f1e0237c0a5d25456987a3d4514177e24 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-4-deactivate, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240326, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.33.5, ceph=True, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-03-26T21:13:19.295 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:13:19 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6[62438]: 2024-03-26T21:13:19.103+0000 7f05ca911700 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:43.011095+0000 front 2024-03-26T21:12:43.010968+0000 (oldest deadline 2024-03-26T21:13:05.911029+0000) 2024-03-26T21:13:19.295 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:13:19 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6[62438]: 2024-03-26T21:13:19.103+0000 7f05ca911700 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-03-26T21:12:47.612134+0000 front 2024-03-26T21:12:47.612161+0000 (oldest deadline 2024-03-26T21:13:13.512031+0000) 2024-03-26T21:13:19.295 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:13:19 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-5[60157]: 2024-03-26T21:13:19.039+0000 7f2813359700 -1 osd.5 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:41.335725+0000 front 2024-03-26T21:12:41.335743+0000 (oldest deadline 2024-03-26T21:13:06.035916+0000) 2024-03-26T21:13:19.295 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:13:19 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-5[60157]: 2024-03-26T21:13:19.039+0000 7f2813359700 -1 osd.5 157 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-03-26T21:12:52.337565+0000 front 2024-03-26T21:12:52.337707+0000 (oldest deadline 2024-03-26T21:13:18.237621+0000) 2024-03-26T21:13:19.295 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:19 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:19.150+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:41.080008+0000 front 2024-03-26T21:12:41.080165+0000 (oldest deadline 2024-03-26T21:13:08.680479+0000) 2024-03-26T21:13:19.296 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:19 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:19.150+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-03-26T21:12:48.680857+0000 front 2024-03-26T21:12:48.680883+0000 (oldest deadline 2024-03-26T21:13:13.981129+0000) 2024-03-26T21:13:19.633 DEBUG:teuthology.orchestra.run.smithi174:> sudo pkill -f 'journalctl -f -n 0 -u ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715@osd.4.service' 2024-03-26T21:13:19.714 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-26T21:13:19.714 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-03-26T21:13:19.715 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-03-26T21:13:19.715 DEBUG:teuthology.orchestra.run.smithi174:> sudo systemctl stop ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715@osd.5 2024-03-26T21:13:20.031 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:13:19 smithi174 systemd[1]: Stopping Ceph osd.5 for 0ed4dd4c-ebb2-11ee-95d0-87774f69a715... 2024-03-26T21:13:20.031 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:13:20 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-5[60157]: 2024-03-26T21:13:20.037+0000 7f2813359700 -1 osd.5 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:41.335725+0000 front 2024-03-26T21:12:41.335743+0000 (oldest deadline 2024-03-26T21:13:06.035916+0000) 2024-03-26T21:13:20.032 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:13:20 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-5[60157]: 2024-03-26T21:13:20.037+0000 7f2813359700 -1 osd.5 157 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-03-26T21:12:52.337565+0000 front 2024-03-26T21:12:52.337707+0000 (oldest deadline 2024-03-26T21:13:18.237621+0000) 2024-03-26T21:13:20.298 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:13:20 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6[62438]: 2024-03-26T21:13:20.123+0000 7f05ca911700 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:43.011095+0000 front 2024-03-26T21:12:43.010968+0000 (oldest deadline 2024-03-26T21:13:05.911029+0000) 2024-03-26T21:13:20.298 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:13:20 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6[62438]: 2024-03-26T21:13:20.123+0000 7f05ca911700 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-03-26T21:12:47.612134+0000 front 2024-03-26T21:12:47.612161+0000 (oldest deadline 2024-03-26T21:13:13.512031+0000) 2024-03-26T21:13:20.298 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:20 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:20.161+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:41.080008+0000 front 2024-03-26T21:12:41.080165+0000 (oldest deadline 2024-03-26T21:13:08.680479+0000) 2024-03-26T21:13:20.299 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:20 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:20.161+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-03-26T21:12:48.680857+0000 front 2024-03-26T21:12:48.680883+0000 (oldest deadline 2024-03-26T21:13:13.981129+0000) 2024-03-26T21:13:20.299 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:13:20 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-5[60157]: 2024-03-26T21:13:20.071+0000 7f281ad6d700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-26T21:13:20.299 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:13:20 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-5[60157]: 2024-03-26T21:13:20.071+0000 7f281ad6d700 -1 osd.5 157 *** Got signal Terminated *** 2024-03-26T21:13:20.299 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:13:20 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-5[60157]: 2024-03-26T21:13:20.071+0000 7f281ad6d700 -1 osd.5 157 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-26T21:13:21.548 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:13:21 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6[62438]: 2024-03-26T21:13:21.109+0000 7f05ca911700 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:43.011095+0000 front 2024-03-26T21:12:43.010968+0000 (oldest deadline 2024-03-26T21:13:05.911029+0000) 2024-03-26T21:13:21.548 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:13:21 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6[62438]: 2024-03-26T21:13:21.109+0000 7f05ca911700 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-03-26T21:12:47.612134+0000 front 2024-03-26T21:12:47.612161+0000 (oldest deadline 2024-03-26T21:13:13.512031+0000) 2024-03-26T21:13:21.548 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:13:21 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-5[60157]: 2024-03-26T21:13:21.082+0000 7f2813359700 -1 osd.5 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:41.335725+0000 front 2024-03-26T21:12:41.335743+0000 (oldest deadline 2024-03-26T21:13:06.035916+0000) 2024-03-26T21:13:21.549 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:13:21 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-5[60157]: 2024-03-26T21:13:21.082+0000 7f2813359700 -1 osd.5 157 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-03-26T21:12:52.337565+0000 front 2024-03-26T21:12:52.337707+0000 (oldest deadline 2024-03-26T21:13:18.237621+0000) 2024-03-26T21:13:21.549 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:21 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:21.126+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:41.080008+0000 front 2024-03-26T21:12:41.080165+0000 (oldest deadline 2024-03-26T21:13:08.680479+0000) 2024-03-26T21:13:21.549 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:21 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:21.126+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-03-26T21:12:48.680857+0000 front 2024-03-26T21:12:48.680883+0000 (oldest deadline 2024-03-26T21:13:13.981129+0000) 2024-03-26T21:13:22.548 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:13:22 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-5[60157]: 2024-03-26T21:13:22.077+0000 7f2813359700 -1 osd.5 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:41.335725+0000 front 2024-03-26T21:12:41.335743+0000 (oldest deadline 2024-03-26T21:13:06.035916+0000) 2024-03-26T21:13:22.548 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:13:22 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-5[60157]: 2024-03-26T21:13:22.077+0000 7f2813359700 -1 osd.5 157 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-03-26T21:12:52.337565+0000 front 2024-03-26T21:12:52.337707+0000 (oldest deadline 2024-03-26T21:13:18.237621+0000) 2024-03-26T21:13:22.548 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:13:22 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6[62438]: 2024-03-26T21:13:22.118+0000 7f05ca911700 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:43.011095+0000 front 2024-03-26T21:12:43.010968+0000 (oldest deadline 2024-03-26T21:13:05.911029+0000) 2024-03-26T21:13:22.549 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:13:22 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6[62438]: 2024-03-26T21:13:22.118+0000 7f05ca911700 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-03-26T21:12:47.612134+0000 front 2024-03-26T21:12:47.612161+0000 (oldest deadline 2024-03-26T21:13:13.512031+0000) 2024-03-26T21:13:22.549 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:22 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:22.095+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:41.080008+0000 front 2024-03-26T21:12:41.080165+0000 (oldest deadline 2024-03-26T21:13:08.680479+0000) 2024-03-26T21:13:22.549 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:22 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:22.095+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-03-26T21:12:48.680857+0000 front 2024-03-26T21:12:48.680883+0000 (oldest deadline 2024-03-26T21:13:13.981129+0000) 2024-03-26T21:13:23.548 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:13:23 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6[62438]: 2024-03-26T21:13:23.139+0000 7f05ca911700 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:43.011095+0000 front 2024-03-26T21:12:43.010968+0000 (oldest deadline 2024-03-26T21:13:05.911029+0000) 2024-03-26T21:13:23.548 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:13:23 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6[62438]: 2024-03-26T21:13:23.139+0000 7f05ca911700 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-03-26T21:12:47.612134+0000 front 2024-03-26T21:12:47.612161+0000 (oldest deadline 2024-03-26T21:13:13.512031+0000) 2024-03-26T21:13:23.548 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:23 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:23.069+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:41.080008+0000 front 2024-03-26T21:12:41.080165+0000 (oldest deadline 2024-03-26T21:13:08.680479+0000) 2024-03-26T21:13:23.548 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:23 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:23.069+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-03-26T21:12:48.680857+0000 front 2024-03-26T21:12:48.680883+0000 (oldest deadline 2024-03-26T21:13:13.981129+0000) 2024-03-26T21:13:23.549 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:13:23 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-5[60157]: 2024-03-26T21:13:23.074+0000 7f2813359700 -1 osd.5 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:41.335725+0000 front 2024-03-26T21:12:41.335743+0000 (oldest deadline 2024-03-26T21:13:06.035916+0000) 2024-03-26T21:13:23.549 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:13:23 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-5[60157]: 2024-03-26T21:13:23.074+0000 7f2813359700 -1 osd.5 157 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-03-26T21:12:52.337565+0000 front 2024-03-26T21:12:52.337707+0000 (oldest deadline 2024-03-26T21:13:18.237621+0000) 2024-03-26T21:13:23.549 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:13:23 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-5[60157]: 2024-03-26T21:13:23.074+0000 7f2813359700 -1 osd.5 157 heartbeat_check: no reply from 172.21.15.63:6822 osd.2 since back 2024-03-26T21:12:58.238053+0000 front 2024-03-26T21:12:58.238016+0000 (oldest deadline 2024-03-26T21:13:22.938167+0000) 2024-03-26T21:13:24.548 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:13:24 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6[62438]: 2024-03-26T21:13:24.128+0000 7f05ca911700 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:43.011095+0000 front 2024-03-26T21:12:43.010968+0000 (oldest deadline 2024-03-26T21:13:05.911029+0000) 2024-03-26T21:13:24.548 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:13:24 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6[62438]: 2024-03-26T21:13:24.128+0000 7f05ca911700 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-03-26T21:12:47.612134+0000 front 2024-03-26T21:12:47.612161+0000 (oldest deadline 2024-03-26T21:13:13.512031+0000) 2024-03-26T21:13:24.549 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:24 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:24.107+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:41.080008+0000 front 2024-03-26T21:12:41.080165+0000 (oldest deadline 2024-03-26T21:13:08.680479+0000) 2024-03-26T21:13:24.549 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:24 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:24.107+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-03-26T21:12:48.680857+0000 front 2024-03-26T21:12:48.680883+0000 (oldest deadline 2024-03-26T21:13:13.981129+0000) 2024-03-26T21:13:24.549 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:24 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:24.107+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6822 osd.2 since back 2024-03-26T21:12:59.182751+0000 front 2024-03-26T21:12:59.182736+0000 (oldest deadline 2024-03-26T21:13:23.882758+0000) 2024-03-26T21:13:24.549 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:13:24 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-5[60157]: 2024-03-26T21:13:24.094+0000 7f2813359700 -1 osd.5 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:41.335725+0000 front 2024-03-26T21:12:41.335743+0000 (oldest deadline 2024-03-26T21:13:06.035916+0000) 2024-03-26T21:13:24.549 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:13:24 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-5[60157]: 2024-03-26T21:13:24.094+0000 7f2813359700 -1 osd.5 157 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-03-26T21:12:52.337565+0000 front 2024-03-26T21:12:52.337707+0000 (oldest deadline 2024-03-26T21:13:18.237621+0000) 2024-03-26T21:13:24.549 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:13:24 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-5[60157]: 2024-03-26T21:13:24.094+0000 7f2813359700 -1 osd.5 157 heartbeat_check: no reply from 172.21.15.63:6822 osd.2 since back 2024-03-26T21:12:58.238053+0000 front 2024-03-26T21:12:58.238016+0000 (oldest deadline 2024-03-26T21:13:22.938167+0000) 2024-03-26T21:13:25.533 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:13:25 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6[62438]: 2024-03-26T21:13:25.095+0000 7f05ca911700 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:43.011095+0000 front 2024-03-26T21:12:43.010968+0000 (oldest deadline 2024-03-26T21:13:05.911029+0000) 2024-03-26T21:13:25.534 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:13:25 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6[62438]: 2024-03-26T21:13:25.095+0000 7f05ca911700 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-03-26T21:12:47.612134+0000 front 2024-03-26T21:12:47.612161+0000 (oldest deadline 2024-03-26T21:13:13.512031+0000) 2024-03-26T21:13:25.534 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:25 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:25.112+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:41.080008+0000 front 2024-03-26T21:12:41.080165+0000 (oldest deadline 2024-03-26T21:13:08.680479+0000) 2024-03-26T21:13:25.534 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:25 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:25.113+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-03-26T21:12:48.680857+0000 front 2024-03-26T21:12:48.680883+0000 (oldest deadline 2024-03-26T21:13:13.981129+0000) 2024-03-26T21:13:25.534 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:25 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:25.113+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6822 osd.2 since back 2024-03-26T21:12:59.182751+0000 front 2024-03-26T21:12:59.182736+0000 (oldest deadline 2024-03-26T21:13:23.882758+0000) 2024-03-26T21:13:25.798 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:13:25 smithi174 podman[79152]: 2024-03-26 21:13:25.539959595 +0000 UTC m=+5.596943930 container died 25dde01e0676876c4bf2abbdea177dabaeca8edb525a42040e9b1f434af7e9cd (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-5, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240326, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-03-26T21:13:26.081 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:13:25 smithi174 podman[79152]: 2024-03-26 21:13:25.895524476 +0000 UTC m=+5.952508798 container cleanup 25dde01e0676876c4bf2abbdea177dabaeca8edb525a42040e9b1f434af7e9cd (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-5, org.label-schema.build-date=20240326, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, CEPH_POINT_RELEASE=) 2024-03-26T21:13:26.081 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:13:25 smithi174 bash[79152]: ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-5 2024-03-26T21:13:26.500 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:13:26 smithi174 podman[79202]: 2024-03-26 21:13:26.180306045 +0000 UTC m=+0.635147055 container remove 25dde01e0676876c4bf2abbdea177dabaeca8edb525a42040e9b1f434af7e9cd (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-5, ceph=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, org.label-schema.build-date=20240326, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-26T21:13:26.500 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:26 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:26.129+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:41.080008+0000 front 2024-03-26T21:12:41.080165+0000 (oldest deadline 2024-03-26T21:13:08.680479+0000) 2024-03-26T21:13:26.500 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:26 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:26.129+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-03-26T21:12:48.680857+0000 front 2024-03-26T21:12:48.680883+0000 (oldest deadline 2024-03-26T21:13:13.981129+0000) 2024-03-26T21:13:26.500 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:26 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:26.129+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6822 osd.2 since back 2024-03-26T21:12:59.182751+0000 front 2024-03-26T21:12:59.182736+0000 (oldest deadline 2024-03-26T21:13:23.882758+0000) 2024-03-26T21:13:26.500 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:13:26 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6[62438]: 2024-03-26T21:13:26.086+0000 7f05ca911700 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:43.011095+0000 front 2024-03-26T21:12:43.010968+0000 (oldest deadline 2024-03-26T21:13:05.911029+0000) 2024-03-26T21:13:26.501 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:13:26 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6[62438]: 2024-03-26T21:13:26.086+0000 7f05ca911700 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-03-26T21:12:47.612134+0000 front 2024-03-26T21:12:47.612161+0000 (oldest deadline 2024-03-26T21:13:13.512031+0000) 2024-03-26T21:13:26.501 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:13:26 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6[62438]: 2024-03-26T21:13:26.086+0000 7f05ca911700 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.63:6822 osd.2 since back 2024-03-26T21:12:59.313338+0000 front 2024-03-26T21:12:59.313216+0000 (oldest deadline 2024-03-26T21:13:25.213234+0000) 2024-03-26T21:13:26.785 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:13:26 smithi174 podman[79296]: 2024-03-26 21:13:26.409323715 +0000 UTC m=+0.017522308 image pull a9bdc3eaeb106993bcbbac35d7fe5c7b76c052021676e129f5984aed30aee063 quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d 2024-03-26T21:13:26.785 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:13:26 smithi174 podman[79296]: 2024-03-26 21:13:26.514330203 +0000 UTC m=+0.122528782 container create 873d8383221bfd96bcf763a5d01b308743a2d7fdd398535c19d8ecf62eb56cfb (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-5-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240326, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-03-26T21:13:26.785 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:13:26 smithi174 podman[79296]: 2024-03-26 21:13:26.717746436 +0000 UTC m=+0.325945021 container init 873d8383221bfd96bcf763a5d01b308743a2d7fdd398535c19d8ecf62eb56cfb (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-5-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, maintainer=Guillaume Abrioux , ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.33.5) 2024-03-26T21:13:26.785 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:13:26 smithi174 podman[79296]: 2024-03-26 21:13:26.720878337 +0000 UTC m=+0.329076922 container start 873d8383221bfd96bcf763a5d01b308743a2d7fdd398535c19d8ecf62eb56cfb (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-5-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-03-26T21:13:26.785 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:13:26 smithi174 podman[79296]: 2024-03-26 21:13:26.791716569 +0000 UTC m=+0.399915165 container attach 873d8383221bfd96bcf763a5d01b308743a2d7fdd398535c19d8ecf62eb56cfb (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-5-deactivate, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-26T21:13:27.048 INFO:journalctl@ceph.osd.5.smithi174.stdout:Mar 26 21:13:27 smithi174 podman[79296]: 2024-03-26 21:13:27.01936977 +0000 UTC m=+0.627568340 container died 873d8383221bfd96bcf763a5d01b308743a2d7fdd398535c19d8ecf62eb56cfb (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-5-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2) 2024-03-26T21:13:27.548 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:13:27 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6[62438]: 2024-03-26T21:13:27.070+0000 7f05ca911700 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:43.011095+0000 front 2024-03-26T21:12:43.010968+0000 (oldest deadline 2024-03-26T21:13:05.911029+0000) 2024-03-26T21:13:27.548 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:13:27 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6[62438]: 2024-03-26T21:13:27.070+0000 7f05ca911700 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-03-26T21:12:47.612134+0000 front 2024-03-26T21:12:47.612161+0000 (oldest deadline 2024-03-26T21:13:13.512031+0000) 2024-03-26T21:13:27.548 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:13:27 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6[62438]: 2024-03-26T21:13:27.070+0000 7f05ca911700 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.63:6822 osd.2 since back 2024-03-26T21:12:59.313338+0000 front 2024-03-26T21:12:59.313216+0000 (oldest deadline 2024-03-26T21:13:25.213234+0000) 2024-03-26T21:13:27.549 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:27 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:27.142+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:41.080008+0000 front 2024-03-26T21:12:41.080165+0000 (oldest deadline 2024-03-26T21:13:08.680479+0000) 2024-03-26T21:13:27.549 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:27 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:27.142+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-03-26T21:12:48.680857+0000 front 2024-03-26T21:12:48.680883+0000 (oldest deadline 2024-03-26T21:13:13.981129+0000) 2024-03-26T21:13:27.549 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:27 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:27.142+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6822 osd.2 since back 2024-03-26T21:12:59.182751+0000 front 2024-03-26T21:12:59.182736+0000 (oldest deadline 2024-03-26T21:13:23.882758+0000) 2024-03-26T21:13:27.607 DEBUG:teuthology.orchestra.run.smithi174:> sudo pkill -f 'journalctl -f -n 0 -u ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715@osd.5.service' 2024-03-26T21:13:27.652 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-26T21:13:27.652 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-03-26T21:13:27.652 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-03-26T21:13:27.652 DEBUG:teuthology.orchestra.run.smithi174:> sudo systemctl stop ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715@osd.6 2024-03-26T21:13:27.942 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:13:27 smithi174 systemd[1]: Stopping Ceph osd.6 for 0ed4dd4c-ebb2-11ee-95d0-87774f69a715... 2024-03-26T21:13:27.942 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:13:27 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6[62438]: 2024-03-26T21:13:27.947+0000 7f05ce31d700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-26T21:13:27.942 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:13:27 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6[62438]: 2024-03-26T21:13:27.947+0000 7f05ce31d700 -1 osd.6 157 *** Got signal Terminated *** 2024-03-26T21:13:27.942 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:13:27 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6[62438]: 2024-03-26T21:13:27.947+0000 7f05ce31d700 -1 osd.6 157 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-26T21:13:28.298 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:13:28 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6[62438]: 2024-03-26T21:13:28.027+0000 7f05ca911700 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:43.011095+0000 front 2024-03-26T21:12:43.010968+0000 (oldest deadline 2024-03-26T21:13:05.911029+0000) 2024-03-26T21:13:28.298 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:13:28 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6[62438]: 2024-03-26T21:13:28.027+0000 7f05ca911700 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-03-26T21:12:47.612134+0000 front 2024-03-26T21:12:47.612161+0000 (oldest deadline 2024-03-26T21:13:13.512031+0000) 2024-03-26T21:13:28.298 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:13:28 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6[62438]: 2024-03-26T21:13:28.027+0000 7f05ca911700 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.63:6822 osd.2 since back 2024-03-26T21:12:59.313338+0000 front 2024-03-26T21:12:59.313216+0000 (oldest deadline 2024-03-26T21:13:25.213234+0000) 2024-03-26T21:13:28.298 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:28 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:28.186+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:41.080008+0000 front 2024-03-26T21:12:41.080165+0000 (oldest deadline 2024-03-26T21:13:08.680479+0000) 2024-03-26T21:13:28.298 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:28 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:28.186+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-03-26T21:12:48.680857+0000 front 2024-03-26T21:12:48.680883+0000 (oldest deadline 2024-03-26T21:13:13.981129+0000) 2024-03-26T21:13:28.298 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:28 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:28.186+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6822 osd.2 since back 2024-03-26T21:12:59.182751+0000 front 2024-03-26T21:12:59.182736+0000 (oldest deadline 2024-03-26T21:13:23.882758+0000) 2024-03-26T21:13:29.298 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:13:28 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6[62438]: 2024-03-26T21:13:28.986+0000 7f05ca911700 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:43.011095+0000 front 2024-03-26T21:12:43.010968+0000 (oldest deadline 2024-03-26T21:13:05.911029+0000) 2024-03-26T21:13:29.298 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:13:28 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6[62438]: 2024-03-26T21:13:28.986+0000 7f05ca911700 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-03-26T21:12:47.612134+0000 front 2024-03-26T21:12:47.612161+0000 (oldest deadline 2024-03-26T21:13:13.512031+0000) 2024-03-26T21:13:29.298 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:13:28 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6[62438]: 2024-03-26T21:13:28.986+0000 7f05ca911700 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.63:6822 osd.2 since back 2024-03-26T21:12:59.313338+0000 front 2024-03-26T21:12:59.313216+0000 (oldest deadline 2024-03-26T21:13:25.213234+0000) 2024-03-26T21:13:29.299 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:29 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:29.213+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:41.080008+0000 front 2024-03-26T21:12:41.080165+0000 (oldest deadline 2024-03-26T21:13:08.680479+0000) 2024-03-26T21:13:29.299 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:29 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:29.213+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-03-26T21:12:48.680857+0000 front 2024-03-26T21:12:48.680883+0000 (oldest deadline 2024-03-26T21:13:13.981129+0000) 2024-03-26T21:13:29.299 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:29 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:29.213+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6822 osd.2 since back 2024-03-26T21:12:59.182751+0000 front 2024-03-26T21:12:59.182736+0000 (oldest deadline 2024-03-26T21:13:23.882758+0000) 2024-03-26T21:13:30.225 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:13:29 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6[62438]: 2024-03-26T21:13:29.939+0000 7f05ca911700 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:43.011095+0000 front 2024-03-26T21:12:43.010968+0000 (oldest deadline 2024-03-26T21:13:05.911029+0000) 2024-03-26T21:13:30.225 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:13:29 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6[62438]: 2024-03-26T21:13:29.939+0000 7f05ca911700 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-03-26T21:12:47.612134+0000 front 2024-03-26T21:12:47.612161+0000 (oldest deadline 2024-03-26T21:13:13.512031+0000) 2024-03-26T21:13:30.225 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:13:29 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6[62438]: 2024-03-26T21:13:29.939+0000 7f05ca911700 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.63:6822 osd.2 since back 2024-03-26T21:12:59.313338+0000 front 2024-03-26T21:12:59.313216+0000 (oldest deadline 2024-03-26T21:13:25.213234+0000) 2024-03-26T21:13:30.548 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:30 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:30.231+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:41.080008+0000 front 2024-03-26T21:12:41.080165+0000 (oldest deadline 2024-03-26T21:13:08.680479+0000) 2024-03-26T21:13:30.548 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:30 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:30.231+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-03-26T21:12:48.680857+0000 front 2024-03-26T21:12:48.680883+0000 (oldest deadline 2024-03-26T21:13:13.981129+0000) 2024-03-26T21:13:30.548 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:30 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:30.231+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6822 osd.2 since back 2024-03-26T21:12:59.182751+0000 front 2024-03-26T21:12:59.182736+0000 (oldest deadline 2024-03-26T21:13:23.882758+0000) 2024-03-26T21:13:30.548 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:30 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:30.231+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6830 osd.3 since back 2024-03-26T21:13:06.083934+0000 front 2024-03-26T21:13:06.084082+0000 (oldest deadline 2024-03-26T21:13:30.184041+0000) 2024-03-26T21:13:31.223 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:13:30 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6[62438]: 2024-03-26T21:13:30.894+0000 7f05ca911700 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:43.011095+0000 front 2024-03-26T21:12:43.010968+0000 (oldest deadline 2024-03-26T21:13:05.911029+0000) 2024-03-26T21:13:31.223 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:13:30 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6[62438]: 2024-03-26T21:13:30.894+0000 7f05ca911700 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-03-26T21:12:47.612134+0000 front 2024-03-26T21:12:47.612161+0000 (oldest deadline 2024-03-26T21:13:13.512031+0000) 2024-03-26T21:13:31.223 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:13:30 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6[62438]: 2024-03-26T21:13:30.894+0000 7f05ca911700 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.63:6822 osd.2 since back 2024-03-26T21:12:59.313338+0000 front 2024-03-26T21:12:59.313216+0000 (oldest deadline 2024-03-26T21:13:25.213234+0000) 2024-03-26T21:13:31.548 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:31 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:31.229+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:41.080008+0000 front 2024-03-26T21:12:41.080165+0000 (oldest deadline 2024-03-26T21:13:08.680479+0000) 2024-03-26T21:13:31.548 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:31 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:31.229+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-03-26T21:12:48.680857+0000 front 2024-03-26T21:12:48.680883+0000 (oldest deadline 2024-03-26T21:13:13.981129+0000) 2024-03-26T21:13:31.548 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:31 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:31.229+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6822 osd.2 since back 2024-03-26T21:12:59.182751+0000 front 2024-03-26T21:12:59.182736+0000 (oldest deadline 2024-03-26T21:13:23.882758+0000) 2024-03-26T21:13:31.548 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:31 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:31.229+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6830 osd.3 since back 2024-03-26T21:13:06.083934+0000 front 2024-03-26T21:13:06.084082+0000 (oldest deadline 2024-03-26T21:13:30.184041+0000) 2024-03-26T21:13:32.191 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:13:31 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6[62438]: 2024-03-26T21:13:31.936+0000 7f05ca911700 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:43.011095+0000 front 2024-03-26T21:12:43.010968+0000 (oldest deadline 2024-03-26T21:13:05.911029+0000) 2024-03-26T21:13:32.192 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:13:31 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6[62438]: 2024-03-26T21:13:31.936+0000 7f05ca911700 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-03-26T21:12:47.612134+0000 front 2024-03-26T21:12:47.612161+0000 (oldest deadline 2024-03-26T21:13:13.512031+0000) 2024-03-26T21:13:32.192 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:13:31 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6[62438]: 2024-03-26T21:13:31.936+0000 7f05ca911700 -1 osd.6 157 heartbeat_check: no reply from 172.21.15.63:6822 osd.2 since back 2024-03-26T21:12:59.313338+0000 front 2024-03-26T21:12:59.313216+0000 (oldest deadline 2024-03-26T21:13:25.213234+0000) 2024-03-26T21:13:32.548 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:32 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:32.197+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:41.080008+0000 front 2024-03-26T21:12:41.080165+0000 (oldest deadline 2024-03-26T21:13:08.680479+0000) 2024-03-26T21:13:32.548 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:32 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:32.197+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-03-26T21:12:48.680857+0000 front 2024-03-26T21:12:48.680883+0000 (oldest deadline 2024-03-26T21:13:13.981129+0000) 2024-03-26T21:13:32.548 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:32 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:32.197+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6822 osd.2 since back 2024-03-26T21:12:59.182751+0000 front 2024-03-26T21:12:59.182736+0000 (oldest deadline 2024-03-26T21:13:23.882758+0000) 2024-03-26T21:13:32.548 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:32 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:32.197+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6830 osd.3 since back 2024-03-26T21:13:06.083934+0000 front 2024-03-26T21:13:06.084082+0000 (oldest deadline 2024-03-26T21:13:30.184041+0000) 2024-03-26T21:13:33.548 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:13:33 smithi174 podman[79401]: 2024-03-26 21:13:33.38170526 +0000 UTC m=+5.539926691 container died 95fa1b77701bf489b9459a0b0b93c37fe1c09c89a50da6d183a6fbee6d99b625 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240326, io.buildah.version=1.33.5) 2024-03-26T21:13:33.548 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:33 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:33.231+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:41.080008+0000 front 2024-03-26T21:12:41.080165+0000 (oldest deadline 2024-03-26T21:13:08.680479+0000) 2024-03-26T21:13:33.548 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:33 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:33.231+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-03-26T21:12:48.680857+0000 front 2024-03-26T21:12:48.680883+0000 (oldest deadline 2024-03-26T21:13:13.981129+0000) 2024-03-26T21:13:33.548 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:33 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:33.231+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6822 osd.2 since back 2024-03-26T21:12:59.182751+0000 front 2024-03-26T21:12:59.182736+0000 (oldest deadline 2024-03-26T21:13:23.882758+0000) 2024-03-26T21:13:33.548 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:33 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:33.231+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6830 osd.3 since back 2024-03-26T21:13:06.083934+0000 front 2024-03-26T21:13:06.084082+0000 (oldest deadline 2024-03-26T21:13:30.184041+0000) 2024-03-26T21:13:33.984 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:13:33 smithi174 podman[79401]: 2024-03-26 21:13:33.756285904 +0000 UTC m=+5.914507322 container cleanup 95fa1b77701bf489b9459a0b0b93c37fe1c09c89a50da6d183a6fbee6d99b625 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6, io.buildah.version=1.33.5, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, ceph=True) 2024-03-26T21:13:33.984 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:13:33 smithi174 bash[79401]: ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6 2024-03-26T21:13:34.298 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:13:33 smithi174 podman[79418]: 2024-03-26 21:13:33.99052526 +0000 UTC m=+0.605914758 container remove 95fa1b77701bf489b9459a0b0b93c37fe1c09c89a50da6d183a6fbee6d99b625 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True, GIT_BRANCH=HEAD) 2024-03-26T21:13:34.299 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:34 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:34.201+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:41.080008+0000 front 2024-03-26T21:12:41.080165+0000 (oldest deadline 2024-03-26T21:13:08.680479+0000) 2024-03-26T21:13:34.299 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:34 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:34.201+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-03-26T21:12:48.680857+0000 front 2024-03-26T21:12:48.680883+0000 (oldest deadline 2024-03-26T21:13:13.981129+0000) 2024-03-26T21:13:34.299 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:34 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:34.201+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6822 osd.2 since back 2024-03-26T21:12:59.182751+0000 front 2024-03-26T21:12:59.182736+0000 (oldest deadline 2024-03-26T21:13:23.882758+0000) 2024-03-26T21:13:34.299 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:34 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:34.201+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6830 osd.3 since back 2024-03-26T21:13:06.083934+0000 front 2024-03-26T21:13:06.084082+0000 (oldest deadline 2024-03-26T21:13:30.184041+0000) 2024-03-26T21:13:34.653 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:13:34 smithi174 podman[79508]: 2024-03-26 21:13:34.224101216 +0000 UTC m=+0.017799590 image pull a9bdc3eaeb106993bcbbac35d7fe5c7b76c052021676e129f5984aed30aee063 quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d 2024-03-26T21:13:34.654 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:13:34 smithi174 podman[79508]: 2024-03-26 21:13:34.349717831 +0000 UTC m=+0.143416177 container create dbdf43211fe5d3881d0e740f8d73c252bb5455a5cb61df08a622ee292722509c (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6-deactivate, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240326, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b) 2024-03-26T21:13:34.654 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:13:34 smithi174 podman[79508]: 2024-03-26 21:13:34.560454522 +0000 UTC m=+0.354152868 container init dbdf43211fe5d3881d0e740f8d73c252bb5455a5cb61df08a622ee292722509c (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6-deactivate, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-26T21:13:34.654 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:13:34 smithi174 podman[79508]: 2024-03-26 21:13:34.563524558 +0000 UTC m=+0.357222907 container start dbdf43211fe5d3881d0e740f8d73c252bb5455a5cb61df08a622ee292722509c (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6-deactivate, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240326, ceph=True, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-03-26T21:13:34.654 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:13:34 smithi174 podman[79508]: 2024-03-26 21:13:34.660499143 +0000 UTC m=+0.454197507 container attach dbdf43211fe5d3881d0e740f8d73c252bb5455a5cb61df08a622ee292722509c (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6-deactivate, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, ceph=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-03-26T21:13:35.048 INFO:journalctl@ceph.osd.6.smithi174.stdout:Mar 26 21:13:34 smithi174 podman[79508]: 2024-03-26 21:13:34.855676275 +0000 UTC m=+0.649374640 container died dbdf43211fe5d3881d0e740f8d73c252bb5455a5cb61df08a622ee292722509c (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-6-deactivate, org.label-schema.schema-version=1.0, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, org.label-schema.license=GPLv2, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD) 2024-03-26T21:13:35.407 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:35 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:35.164+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:41.080008+0000 front 2024-03-26T21:12:41.080165+0000 (oldest deadline 2024-03-26T21:13:08.680479+0000) 2024-03-26T21:13:35.407 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:35 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:35.164+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-03-26T21:12:48.680857+0000 front 2024-03-26T21:12:48.680883+0000 (oldest deadline 2024-03-26T21:13:13.981129+0000) 2024-03-26T21:13:35.407 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:35 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:35.164+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6822 osd.2 since back 2024-03-26T21:12:59.182751+0000 front 2024-03-26T21:12:59.182736+0000 (oldest deadline 2024-03-26T21:13:23.882758+0000) 2024-03-26T21:13:35.407 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:35 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:35.164+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6830 osd.3 since back 2024-03-26T21:13:06.083934+0000 front 2024-03-26T21:13:06.084082+0000 (oldest deadline 2024-03-26T21:13:30.184041+0000) 2024-03-26T21:13:35.425 DEBUG:teuthology.orchestra.run.smithi174:> sudo pkill -f 'journalctl -f -n 0 -u ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715@osd.6.service' 2024-03-26T21:13:35.466 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-26T21:13:35.467 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-03-26T21:13:35.467 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-03-26T21:13:35.467 DEBUG:teuthology.orchestra.run.smithi174:> sudo systemctl stop ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715@osd.7 2024-03-26T21:13:35.794 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:35 smithi174 systemd[1]: Stopping Ceph osd.7 for 0ed4dd4c-ebb2-11ee-95d0-87774f69a715... 2024-03-26T21:13:36.048 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:35 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:35.799+0000 7feb9dbc9700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-26T21:13:36.048 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:35 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:35.799+0000 7feb9dbc9700 -1 osd.7 157 *** Got signal Terminated *** 2024-03-26T21:13:36.048 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:35 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:35.799+0000 7feb9dbc9700 -1 osd.7 157 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-26T21:13:36.548 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:36 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:36.120+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:41.080008+0000 front 2024-03-26T21:12:41.080165+0000 (oldest deadline 2024-03-26T21:13:08.680479+0000) 2024-03-26T21:13:36.548 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:36 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:36.120+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-03-26T21:12:48.680857+0000 front 2024-03-26T21:12:48.680883+0000 (oldest deadline 2024-03-26T21:13:13.981129+0000) 2024-03-26T21:13:36.548 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:36 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:36.120+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6822 osd.2 since back 2024-03-26T21:12:59.182751+0000 front 2024-03-26T21:12:59.182736+0000 (oldest deadline 2024-03-26T21:13:23.882758+0000) 2024-03-26T21:13:36.548 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:36 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:36.120+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6830 osd.3 since back 2024-03-26T21:13:06.083934+0000 front 2024-03-26T21:13:06.084082+0000 (oldest deadline 2024-03-26T21:13:30.184041+0000) 2024-03-26T21:13:37.298 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:37 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:37.116+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:41.080008+0000 front 2024-03-26T21:12:41.080165+0000 (oldest deadline 2024-03-26T21:13:08.680479+0000) 2024-03-26T21:13:37.298 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:37 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:37.116+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-03-26T21:12:48.680857+0000 front 2024-03-26T21:12:48.680883+0000 (oldest deadline 2024-03-26T21:13:13.981129+0000) 2024-03-26T21:13:37.298 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:37 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:37.116+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6822 osd.2 since back 2024-03-26T21:12:59.182751+0000 front 2024-03-26T21:12:59.182736+0000 (oldest deadline 2024-03-26T21:13:23.882758+0000) 2024-03-26T21:13:37.298 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:37 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:37.116+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6830 osd.3 since back 2024-03-26T21:13:06.083934+0000 front 2024-03-26T21:13:06.084082+0000 (oldest deadline 2024-03-26T21:13:30.184041+0000) 2024-03-26T21:13:38.548 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:38 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:38.147+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:41.080008+0000 front 2024-03-26T21:12:41.080165+0000 (oldest deadline 2024-03-26T21:13:08.680479+0000) 2024-03-26T21:13:38.548 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:38 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:38.147+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-03-26T21:12:48.680857+0000 front 2024-03-26T21:12:48.680883+0000 (oldest deadline 2024-03-26T21:13:13.981129+0000) 2024-03-26T21:13:38.548 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:38 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:38.147+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6822 osd.2 since back 2024-03-26T21:12:59.182751+0000 front 2024-03-26T21:12:59.182736+0000 (oldest deadline 2024-03-26T21:13:23.882758+0000) 2024-03-26T21:13:38.548 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:38 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:38.147+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6830 osd.3 since back 2024-03-26T21:13:06.083934+0000 front 2024-03-26T21:13:06.084082+0000 (oldest deadline 2024-03-26T21:13:30.184041+0000) 2024-03-26T21:13:38.548 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:38 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:38.147+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.174:6804 osd.4 since back 2024-03-26T21:13:15.985203+0000 front 2024-03-26T21:13:15.985265+0000 (oldest deadline 2024-03-26T21:13:37.685199+0000) 2024-03-26T21:13:39.548 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:39 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:39.160+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:41.080008+0000 front 2024-03-26T21:12:41.080165+0000 (oldest deadline 2024-03-26T21:13:08.680479+0000) 2024-03-26T21:13:39.548 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:39 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:39.160+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-03-26T21:12:48.680857+0000 front 2024-03-26T21:12:48.680883+0000 (oldest deadline 2024-03-26T21:13:13.981129+0000) 2024-03-26T21:13:39.548 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:39 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:39.160+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6822 osd.2 since back 2024-03-26T21:12:59.182751+0000 front 2024-03-26T21:12:59.182736+0000 (oldest deadline 2024-03-26T21:13:23.882758+0000) 2024-03-26T21:13:39.548 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:39 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:39.160+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6830 osd.3 since back 2024-03-26T21:13:06.083934+0000 front 2024-03-26T21:13:06.084082+0000 (oldest deadline 2024-03-26T21:13:30.184041+0000) 2024-03-26T21:13:39.548 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:39 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:39.160+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.174:6804 osd.4 since back 2024-03-26T21:13:15.985203+0000 front 2024-03-26T21:13:15.985265+0000 (oldest deadline 2024-03-26T21:13:37.685199+0000) 2024-03-26T21:13:40.548 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:40 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:40.210+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-03-26T21:12:41.080008+0000 front 2024-03-26T21:12:41.080165+0000 (oldest deadline 2024-03-26T21:13:08.680479+0000) 2024-03-26T21:13:40.548 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:40 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:40.210+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-03-26T21:12:48.680857+0000 front 2024-03-26T21:12:48.680883+0000 (oldest deadline 2024-03-26T21:13:13.981129+0000) 2024-03-26T21:13:40.548 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:40 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:40.210+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6822 osd.2 since back 2024-03-26T21:12:59.182751+0000 front 2024-03-26T21:12:59.182736+0000 (oldest deadline 2024-03-26T21:13:23.882758+0000) 2024-03-26T21:13:40.548 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:40 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:40.210+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.63:6830 osd.3 since back 2024-03-26T21:13:06.083934+0000 front 2024-03-26T21:13:06.084082+0000 (oldest deadline 2024-03-26T21:13:30.184041+0000) 2024-03-26T21:13:40.548 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:40 smithi174 ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7[64707]: 2024-03-26T21:13:40.210+0000 7feb9a1bd700 -1 osd.7 157 heartbeat_check: no reply from 172.21.15.174:6804 osd.4 since back 2024-03-26T21:13:15.985203+0000 front 2024-03-26T21:13:15.985265+0000 (oldest deadline 2024-03-26T21:13:37.685199+0000) 2024-03-26T21:13:41.548 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:41 smithi174 podman[79614]: 2024-03-26 21:13:41.274361254 +0000 UTC m=+5.580911867 container died cb89961bc93097f840158ac51af5b6a14029f0c99bb1db5ee15ff76bc93b7374 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240326, ceph=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, maintainer=Guillaume Abrioux ) 2024-03-26T21:13:42.048 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:41 smithi174 podman[79614]: 2024-03-26 21:13:41.591305808 +0000 UTC m=+5.897856430 container cleanup cb89961bc93097f840158ac51af5b6a14029f0c99bb1db5ee15ff76bc93b7374 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.build-date=20240326) 2024-03-26T21:13:42.048 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:41 smithi174 bash[79614]: ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7 2024-03-26T21:13:42.048 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:41 smithi174 podman[79676]: 2024-03-26 21:13:41.825604223 +0000 UTC m=+0.545391431 container remove cb89961bc93097f840158ac51af5b6a14029f0c99bb1db5ee15ff76bc93b7374 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7, org.label-schema.build-date=20240326, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, org.label-schema.vendor=CentOS) 2024-03-26T21:13:42.397 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:42 smithi174 podman[79770]: 2024-03-26 21:13:42.045281311 +0000 UTC m=+0.017628300 image pull a9bdc3eaeb106993bcbbac35d7fe5c7b76c052021676e129f5984aed30aee063 quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d 2024-03-26T21:13:42.397 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:42 smithi174 podman[79770]: 2024-03-26 21:13:42.168152638 +0000 UTC m=+0.140499618 container create 5cf209bc70cb28838dfe70ce29aeb85b98bc29ba8a1c67ac59bedb7363220391 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7-deactivate, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS) 2024-03-26T21:13:42.696 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:42 smithi174 podman[79770]: 2024-03-26 21:13:42.404378835 +0000 UTC m=+0.376725830 container init 5cf209bc70cb28838dfe70ce29aeb85b98bc29ba8a1c67ac59bedb7363220391 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7-deactivate, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-03-26T21:13:42.696 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:42 smithi174 podman[79770]: 2024-03-26 21:13:42.407605554 +0000 UTC m=+0.379952543 container start 5cf209bc70cb28838dfe70ce29aeb85b98bc29ba8a1c67ac59bedb7363220391 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7-deactivate, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b) 2024-03-26T21:13:42.696 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:42 smithi174 podman[79770]: 2024-03-26 21:13:42.487352983 +0000 UTC m=+0.459699975 container attach 5cf209bc70cb28838dfe70ce29aeb85b98bc29ba8a1c67ac59bedb7363220391 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7-deactivate, io.buildah.version=1.33.5, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240326, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=) 2024-03-26T21:13:42.984 INFO:journalctl@ceph.osd.7.smithi174.stdout:Mar 26 21:13:42 smithi174 podman[79770]: 2024-03-26 21:13:42.703406259 +0000 UTC m=+0.675753252 container died 5cf209bc70cb28838dfe70ce29aeb85b98bc29ba8a1c67ac59bedb7363220391 (image=quay.ceph.io/ceph-ci/ceph@sha256:77a228078e9cbbda8c0ec34daeaf242ac1c029d4330510ad5bc8624b2b23960d, name=ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715-osd-7-deactivate, RELEASE=wip-guits-testing-2024-03-26-0925-42fc52b, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_COMMIT=d7c7095173c3793ef2ed0fc814eebfb88de11a2e, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS) 2024-03-26T21:13:43.265 DEBUG:teuthology.orchestra.run.smithi174:> sudo pkill -f 'journalctl -f -n 0 -u ceph-0ed4dd4c-ebb2-11ee-95d0-87774f69a715@osd.7.service' 2024-03-26T21:13:43.307 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-26T21:13:43.307 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-03-26T21:13:43.308 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 --force --keep-logs 2024-03-26T21:16:44.592 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 --force --keep-logs 2024-03-26T21:17:06.235 DEBUG:teuthology.orchestra.run.smithi063:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-03-26T21:17:06.272 DEBUG:teuthology.orchestra.run.smithi174:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-03-26T21:17:06.301 INFO:tasks.cephadm:Archiving crash dumps... 2024-03-26T21:17:06.304 DEBUG:teuthology.misc:Transferring archived files from smithi063:/var/lib/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/crash to /home/teuthworker/archive/gabrioux-2024-03-26_20:31:35-orch:cephadm-wip-guits-testing-2024-03-26-0925-distro-default-smithi/7624079/remote/smithi063/crash 2024-03-26T21:17:06.305 DEBUG:teuthology.orchestra.run.smithi063:> sudo tar c -f - -C /var/lib/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/crash -- . 2024-03-26T21:17:06.356 INFO:teuthology.orchestra.run.smithi063.stderr:tar: /var/lib/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/crash: Cannot open: No such file or directory 2024-03-26T21:17:06.356 INFO:teuthology.orchestra.run.smithi063.stderr:tar: Error is not recoverable: exiting now 2024-03-26T21:17:06.358 DEBUG:teuthology.misc:Transferring archived files from smithi174:/var/lib/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/crash to /home/teuthworker/archive/gabrioux-2024-03-26_20:31:35-orch:cephadm-wip-guits-testing-2024-03-26-0925-distro-default-smithi/7624079/remote/smithi174/crash 2024-03-26T21:17:06.359 DEBUG:teuthology.orchestra.run.smithi174:> sudo tar c -f - -C /var/lib/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/crash -- . 2024-03-26T21:17:06.402 INFO:teuthology.orchestra.run.smithi174.stderr:tar: /var/lib/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/crash: Cannot open: No such file or directory 2024-03-26T21:17:06.403 INFO:teuthology.orchestra.run.smithi174.stderr:tar: Error is not recoverable: exiting now 2024-03-26T21:17:06.403 INFO:tasks.cephadm:Checking cluster log for badness... 2024-03-26T21:17:06.404 DEBUG:teuthology.orchestra.run.smithi063:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-03-26T21:17:06.433 INFO:teuthology.orchestra.run.smithi063.stdout:2024-03-26T20:53:30.336493+0000 mon.smithi063 (mon.0) 503 : cluster [WRN] Health check failed: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded (PG_DEGRADED) 2024-03-26T21:17:06.435 WARNING:tasks.cephadm:Found errors (ERR|WRN|SEC) in cluster log 2024-03-26T21:17:06.435 DEBUG:teuthology.orchestra.run.smithi063:> sudo egrep '\[SEC\]' /var/log/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-03-26T21:17:06.507 DEBUG:teuthology.orchestra.run.smithi063:> sudo egrep '\[ERR\]' /var/log/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-03-26T21:17:06.578 DEBUG:teuthology.orchestra.run.smithi063:> sudo egrep '\[WRN\]' /var/log/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-03-26T21:17:06.646 INFO:teuthology.orchestra.run.smithi063.stdout:2024-03-26T20:53:30.336493+0000 mon.smithi063 (mon.0) 503 : cluster [WRN] Health check failed: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded (PG_DEGRADED) 2024-03-26T21:17:06.648 INFO:tasks.cephadm:Compressing logs... 2024-03-26T21:17:06.648 DEBUG:teuthology.orchestra.run.smithi063:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-03-26T21:17:06.690 DEBUG:teuthology.orchestra.run.smithi174:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-03-26T21:17:06.715 INFO:teuthology.orchestra.run.smithi063.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-03-26T21:17:06.715 INFO:teuthology.orchestra.run.smithi063.stderr:: No such file or directory 2024-03-26T21:17:06.715 INFO:teuthology.orchestra.run.smithi063.stderr:gzip -5 --verbose -- /var/log/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/ceph.audit.log 2024-03-26T21:17:06.715 INFO:teuthology.orchestra.run.smithi063.stderr:gzip -5 --verbose -- /var/log/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/ceph-osd.1.log 2024-03-26T21:17:06.717 INFO:teuthology.orchestra.run.smithi063.stderr:/var/log/ceph/cephadm.log: gzip -5 --verbose -- /var/log/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/ceph-osd.3.log 2024-03-26T21:17:06.717 INFO:teuthology.orchestra.run.smithi063.stderr:/var/log/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/ceph.audit.log: /var/log/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/ceph-mon.smithi063.log 2024-03-26T21:17:06.717 INFO:teuthology.orchestra.run.smithi063.stderr:gzip -5 --verbose -- /var/log/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/ceph-mds.foofs.smithi063.tzaznz.log 2024-03-26T21:17:06.717 INFO:teuthology.orchestra.run.smithi063.stderr:/var/log/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/ceph-osd.3.log: /var/log/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/ceph-mon.smithi063.log: gzip -5 --verbose -- /var/log/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/ceph-osd.0.log 2024-03-26T21:17:06.718 INFO:teuthology.orchestra.run.smithi063.stderr:/var/log/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/ceph-mds.foofs.smithi063.tzaznz.log: gzip -5 --verbose -- /var/log/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/ceph.cephadm.log 2024-03-26T21:17:06.718 INFO:teuthology.orchestra.run.smithi063.stderr:/var/log/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/ceph-osd.0.log: gzip -5 --verbose -- /var/log/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/ceph-mgr.smithi063.lxhujt.log 2024-03-26T21:17:06.718 INFO:teuthology.orchestra.run.smithi063.stderr: 89.5% -- replaced with /var/log/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/ceph-mds.foofs.smithi063.tzaznz.log.gz 2024-03-26T21:17:06.720 INFO:teuthology.orchestra.run.smithi063.stderr:/var/log/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/ceph.cephadm.log: 85.7% -- replaced with /var/log/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/ceph.cephadm.log.gz 2024-03-26T21:17:06.720 INFO:teuthology.orchestra.run.smithi063.stderr:gzip -5 --verbose -- /var/log/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/ceph.log 2024-03-26T21:17:06.721 INFO:teuthology.orchestra.run.smithi063.stderr:/var/log/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/ceph-mgr.smithi063.lxhujt.log: gzip -5 --verbose -- /var/log/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/ceph-osd.2.log 2024-03-26T21:17:06.726 INFO:teuthology.orchestra.run.smithi174.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-03-26T21:17:06.726 INFO:teuthology.orchestra.run.smithi174.stderr:: No such file or directory 2024-03-26T21:17:06.726 INFO:teuthology.orchestra.run.smithi174.stderr:gzip -5 --verbose -- /var/log/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/ceph-osd.6.log 2024-03-26T21:17:06.726 INFO:teuthology.orchestra.run.smithi174.stderr:gzip -5 --verbose -- /var/log/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/ceph.audit.log 2024-03-26T21:17:06.726 INFO:teuthology.orchestra.run.smithi174.stderr:/var/log/ceph/cephadm.log: /var/log/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/ceph-osd.6.log: gzip -5 --verbose -- /var/log/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/ceph-mds.foofs.smithi174.gitdvk.log 2024-03-26T21:17:06.727 INFO:teuthology.orchestra.run.smithi174.stderr:/var/log/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/ceph-osd.4.log 2024-03-26T21:17:06.727 INFO:teuthology.orchestra.run.smithi174.stderr:gzip -5 --verbose -- /var/log/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/ceph-mon.smithi174.log 2024-03-26T21:17:06.727 INFO:teuthology.orchestra.run.smithi174.stderr:/var/log/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/ceph-mds.foofs.smithi174.gitdvk.log: /var/log/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/ceph-osd.4.log: gzip -5 --verbose -- /var/log/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/ceph-osd.7.log 2024-03-26T21:17:06.727 INFO:teuthology.orchestra.run.smithi174.stderr: 87.7% -- replaced with /var/log/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/ceph-mds.foofs.smithi174.gitdvk.log.gz 2024-03-26T21:17:06.727 INFO:teuthology.orchestra.run.smithi174.stderr:/var/log/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/ceph-mon.smithi174.log: gzip -5 --verbose -- /var/log/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/ceph-osd.5.log 2024-03-26T21:17:06.727 INFO:teuthology.orchestra.run.smithi174.stderr:/var/log/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/ceph-osd.7.log: gzip -5 --verbose -- /var/log/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/ceph.cephadm.log 2024-03-26T21:17:06.727 INFO:teuthology.orchestra.run.smithi174.stderr:/var/log/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/ceph-osd.5.log: /var/log/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/ceph.cephadm.log: 85.7% -- replaced with /var/log/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/ceph.cephadm.log.gz 2024-03-26T21:17:06.727 INFO:teuthology.orchestra.run.smithi174.stderr:gzip -5 --verbose -- /var/log/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/ceph.log 2024-03-26T21:17:06.727 INFO:teuthology.orchestra.run.smithi174.stderr:gzip -5 --verbose -- /var/log/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/ceph-volume.log 2024-03-26T21:17:06.727 INFO:teuthology.orchestra.run.smithi174.stderr:/var/log/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/ceph.log: 91.7% -- replaced with /var/log/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/ceph.audit.log.gz 2024-03-26T21:17:06.727 INFO:teuthology.orchestra.run.smithi174.stderr:gzip -5 --verbose -- /var/log/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/ceph-mgr.smithi174.sexoin.log 2024-03-26T21:17:06.727 INFO:teuthology.orchestra.run.smithi174.stderr:/var/log/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/ceph-volume.log: 90.2% -- replaced with /var/log/ceph/cephadm.log.gz 2024-03-26T21:17:06.727 INFO:teuthology.orchestra.run.smithi174.stderr:/var/log/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/ceph-mgr.smithi174.sexoin.log: 87.1% -- replaced with /var/log/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/ceph.log.gz 2024-03-26T21:17:06.728 INFO:teuthology.orchestra.run.smithi063.stderr:/var/log/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/ceph.log: 91.6% -- replaced with /var/log/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/ceph.audit.log.gz 2024-03-26T21:17:06.728 INFO:teuthology.orchestra.run.smithi063.stderr:gzip -5 --verbose -- /var/log/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/ceph-volume.log 2024-03-26T21:17:06.728 INFO:teuthology.orchestra.run.smithi063.stderr:/var/log/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/ceph-osd.2.log: 87.2% -- replaced with /var/log/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/ceph.log.gz 2024-03-26T21:17:06.738 INFO:teuthology.orchestra.run.smithi174.stderr: 91.4% -- replaced with /var/log/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/ceph-volume.log.gz 2024-03-26T21:17:06.743 INFO:teuthology.orchestra.run.smithi063.stderr:/var/log/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/ceph-volume.log: 91.3% -- replaced with /var/log/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/ceph-volume.log.gz 2024-03-26T21:17:06.755 INFO:teuthology.orchestra.run.smithi063.stderr: 91.6% -- replaced with /var/log/ceph/cephadm.log.gz 2024-03-26T21:17:06.774 INFO:teuthology.orchestra.run.smithi174.stderr: 89.8% -- replaced with /var/log/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/ceph-mgr.smithi174.sexoin.log.gz 2024-03-26T21:17:07.139 INFO:teuthology.orchestra.run.smithi063.stderr: 88.3% -- replaced with /var/log/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/ceph-mgr.smithi063.lxhujt.log.gz 2024-03-26T21:17:07.219 INFO:teuthology.orchestra.run.smithi174.stderr: 92.6% -- replaced with /var/log/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/ceph-mon.smithi174.log.gz 2024-03-26T21:17:08.078 INFO:teuthology.orchestra.run.smithi063.stderr: 91.9% -- replaced with /var/log/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/ceph-mon.smithi063.log.gz 2024-03-26T21:17:19.276 INFO:teuthology.orchestra.run.smithi063.stderr: 94.5% -- replaced with /var/log/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/ceph-osd.2.log.gz 2024-03-26T21:17:21.898 INFO:teuthology.orchestra.run.smithi174.stderr: 93.9% -- replaced with /var/log/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/ceph-osd.4.log.gz 2024-03-26T21:17:24.126 INFO:teuthology.orchestra.run.smithi174.stderr: 94.2% -- replaced with /var/log/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/ceph-osd.6.log.gz 2024-03-26T21:17:25.145 INFO:teuthology.orchestra.run.smithi063.stderr: 94.1% -- replaced with /var/log/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/ceph-osd.0.log.gz 2024-03-26T21:17:25.559 INFO:teuthology.orchestra.run.smithi063.stderr: 94.2% -- replaced with /var/log/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/ceph-osd.1.log.gz 2024-03-26T21:17:26.830 INFO:teuthology.orchestra.run.smithi174.stderr: 94.2% -- replaced with /var/log/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/ceph-osd.5.log.gz 2024-03-26T21:17:27.813 INFO:teuthology.orchestra.run.smithi063.stderr: 94.1% -- replaced with /var/log/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/ceph-osd.3.log.gz 2024-03-26T21:17:27.815 INFO:teuthology.orchestra.run.smithi063.stderr: 2024-03-26T21:17:27.815 INFO:teuthology.orchestra.run.smithi063.stderr:real 0m21.113s 2024-03-26T21:17:27.815 INFO:teuthology.orchestra.run.smithi063.stderr:user 1m10.146s 2024-03-26T21:17:27.815 INFO:teuthology.orchestra.run.smithi063.stderr:sys 0m2.524s 2024-03-26T21:17:31.616 INFO:teuthology.orchestra.run.smithi174.stderr: 94.1% -- replaced with /var/log/ceph/0ed4dd4c-ebb2-11ee-95d0-87774f69a715/ceph-osd.7.log.gz 2024-03-26T21:17:31.618 INFO:teuthology.orchestra.run.smithi174.stderr: 2024-03-26T21:17:31.618 INFO:teuthology.orchestra.run.smithi174.stderr:real 0m24.913s 2024-03-26T21:17:31.618 INFO:teuthology.orchestra.run.smithi174.stderr:user 1m14.764s 2024-03-26T21:17:31.618 INFO:teuthology.orchestra.run.smithi174.stderr:sys 0m3.056s 2024-03-26T21:17:31.621 INFO:tasks.cephadm:Archiving logs... 2024-03-26T21:17:31.622 DEBUG:teuthology.misc:Transferring archived files from smithi063:/var/log/ceph to /home/teuthworker/archive/gabrioux-2024-03-26_20:31:35-orch:cephadm-wip-guits-testing-2024-03-26-0925-distro-default-smithi/7624079/remote/smithi063/log 2024-03-26T21:17:31.623 DEBUG:teuthology.orchestra.run.smithi063:> sudo tar c -f - -C /var/log/ceph -- . 2024-03-26T21:17:41.402 DEBUG:teuthology.misc:Transferring archived files from smithi174:/var/log/ceph to /home/teuthworker/archive/gabrioux-2024-03-26_20:31:35-orch:cephadm-wip-guits-testing-2024-03-26-0925-distro-default-smithi/7624079/remote/smithi174/log 2024-03-26T21:17:41.403 DEBUG:teuthology.orchestra.run.smithi174:> sudo tar c -f - -C /var/log/ceph -- . 2024-03-26T21:17:52.052 INFO:tasks.cephadm:Removing cluster... 2024-03-26T21:17:52.052 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 --force 2024-03-26T21:17:52.429 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 0ed4dd4c-ebb2-11ee-95d0-87774f69a715 --force 2024-03-26T21:17:52.807 INFO:tasks.cephadm:Removing cephadm ... 2024-03-26T21:17:52.807 DEBUG:teuthology.orchestra.run.smithi063:> rm -rf /home/ubuntu/cephtest/cephadm 2024-03-26T21:17:52.822 DEBUG:teuthology.orchestra.run.smithi174:> rm -rf /home/ubuntu/cephtest/cephadm 2024-03-26T21:17:52.837 INFO:tasks.cephadm:Teardown complete 2024-03-26T21:17:52.837 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-03-26T21:17:52.848 INFO:teuthology.task.clock:Checking final clock skew... 2024-03-26T21:17:52.848 DEBUG:teuthology.orchestra.run.smithi063:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-03-26T21:17:52.865 DEBUG:teuthology.orchestra.run.smithi174:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-03-26T21:17:52.876 INFO:teuthology.orchestra.run.smithi063.stderr:bash: line 1: ntpq: command not found 2024-03-26T21:17:52.880 INFO:teuthology.orchestra.run.smithi063.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-03-26T21:17:52.880 INFO:teuthology.orchestra.run.smithi063.stdout:=============================================================================== 2024-03-26T21:17:52.880 INFO:teuthology.orchestra.run.smithi063.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 8 +512us[ +523us] +/- 94ms 2024-03-26T21:17:52.880 INFO:teuthology.orchestra.run.smithi063.stdout:^* hv02.front.sepia.ceph.com 3 6 377 7 -777us[ -766us] +/- 57ms 2024-03-26T21:17:52.880 INFO:teuthology.orchestra.run.smithi063.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 6 +576us[ +576us] +/- 65ms 2024-03-26T21:17:52.881 INFO:teuthology.orchestra.run.smithi063.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-03-26T21:17:52.890 INFO:teuthology.orchestra.run.smithi174.stderr:bash: line 1: ntpq: command not found 2024-03-26T21:17:52.894 INFO:teuthology.orchestra.run.smithi174.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-03-26T21:17:52.894 INFO:teuthology.orchestra.run.smithi174.stdout:=============================================================================== 2024-03-26T21:17:52.894 INFO:teuthology.orchestra.run.smithi174.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 8 +519us[ +519us] +/- 94ms 2024-03-26T21:17:52.894 INFO:teuthology.orchestra.run.smithi174.stdout:^* hv02.front.sepia.ceph.com 3 6 377 10 -779us[ -739us] +/- 57ms 2024-03-26T21:17:52.894 INFO:teuthology.orchestra.run.smithi174.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 7 +581us[ +581us] +/- 65ms 2024-03-26T21:17:52.894 INFO:teuthology.orchestra.run.smithi174.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-03-26T21:17:52.895 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-03-26T21:17:52.905 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-03-26T21:17:52.906 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-03-26T21:17:52.916 DEBUG:teuthology.orchestra.run.smithi063:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-03-26T21:17:53.005 DEBUG:teuthology.orchestra.run.smithi174:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-03-26T21:17:53.085 DEBUG:teuthology.orchestra.run.smithi063:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-03-26T21:17:53.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-26T21:17:53.322 DEBUG:teuthology.orchestra.run.smithi174:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-03-26T21:17:53.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-26T21:17:53.356 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-03-26T21:17:53.642 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-03-26T21:17:53.697 INFO:teuthology.task.internal:Duration was 2085.605943 seconds 2024-03-26T21:17:53.698 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-03-26T21:17:53.726 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-03-26T21:17:53.726 DEBUG:teuthology.orchestra.run.smithi063:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-03-26T21:17:53.729 DEBUG:teuthology.orchestra.run.smithi174:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-03-26T21:17:53.773 INFO:teuthology.orchestra.run.smithi063.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-03-26T21:17:53.774 INFO:teuthology.orchestra.run.smithi174.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-03-26T21:17:54.095 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-03-26T21:17:54.095 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi063.front.sepia.ceph.com 2024-03-26T21:17:54.095 DEBUG:teuthology.orchestra.run.smithi063:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-03-26T21:17:54.116 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi174.front.sepia.ceph.com 2024-03-26T21:17:54.117 DEBUG:teuthology.orchestra.run.smithi174:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-03-26T21:17:54.149 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-03-26T21:17:54.149 DEBUG:teuthology.orchestra.run.smithi063:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-03-26T21:17:54.159 DEBUG:teuthology.orchestra.run.smithi174:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-03-26T21:17:54.218 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-03-26T21:17:54.218 DEBUG:teuthology.orchestra.run.smithi063:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-03-26T21:17:54.230 DEBUG:teuthology.orchestra.run.smithi174:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-03-26T21:18:08.442 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-03-26T21:18:08.465 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-03-26T21:18:08.466 DEBUG:teuthology.orchestra.run.smithi063:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-03-26T21:18:08.494 DEBUG:teuthology.orchestra.run.smithi174:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-03-26T21:18:08.526 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-03-26T21:18:08.539 DEBUG:teuthology.orchestra.run.smithi063:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-03-26T21:18:08.541 DEBUG:teuthology.orchestra.run.smithi174:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-03-26T21:18:08.563 INFO:teuthology.orchestra.run.smithi063.stdout:kernel.core_pattern = core 2024-03-26T21:18:08.596 INFO:teuthology.orchestra.run.smithi174.stdout:kernel.core_pattern = core 2024-03-26T21:18:08.627 DEBUG:teuthology.orchestra.run.smithi063:> test -e /home/ubuntu/cephtest/archive/coredump 2024-03-26T21:18:08.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-26T21:18:08.652 DEBUG:teuthology.orchestra.run.smithi174:> test -e /home/ubuntu/cephtest/archive/coredump 2024-03-26T21:18:08.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-26T21:18:08.682 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-03-26T21:18:08.693 INFO:teuthology.task.internal:Transferring archived files... 2024-03-26T21:18:08.694 DEBUG:teuthology.misc:Transferring archived files from smithi063:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/gabrioux-2024-03-26_20:31:35-orch:cephadm-wip-guits-testing-2024-03-26-0925-distro-default-smithi/7624079/remote/smithi063 2024-03-26T21:18:08.694 DEBUG:teuthology.orchestra.run.smithi063:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-03-26T21:18:08.758 DEBUG:teuthology.misc:Transferring archived files from smithi174:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/gabrioux-2024-03-26_20:31:35-orch:cephadm-wip-guits-testing-2024-03-26-0925-distro-default-smithi/7624079/remote/smithi174 2024-03-26T21:18:08.758 DEBUG:teuthology.orchestra.run.smithi174:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-03-26T21:18:08.802 INFO:teuthology.task.internal:Removing archive directory... 2024-03-26T21:18:08.802 DEBUG:teuthology.orchestra.run.smithi063:> rm -rf -- /home/ubuntu/cephtest/archive 2024-03-26T21:18:08.805 DEBUG:teuthology.orchestra.run.smithi174:> rm -rf -- /home/ubuntu/cephtest/archive 2024-03-26T21:18:08.852 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-03-26T21:18:08.864 INFO:teuthology.task.internal:Not uploading archives. 2024-03-26T21:18:08.864 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-03-26T21:18:08.900 INFO:teuthology.task.internal:Tidying up after the test... 2024-03-26T21:18:08.900 DEBUG:teuthology.orchestra.run.smithi063:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-03-26T21:18:08.903 DEBUG:teuthology.orchestra.run.smithi174:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-03-26T21:18:08.914 INFO:teuthology.orchestra.run.smithi063.stdout: 262166 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Mar 26 21:18 /home/ubuntu/cephtest 2024-03-26T21:18:08.918 INFO:teuthology.orchestra.run.smithi174.stdout: 262166 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Mar 26 21:18 /home/ubuntu/cephtest 2024-03-26T21:18:08.919 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-03-26T21:18:08.930 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-03-26T21:18:08.952 INFO:teuthology.nuke:Checking targets against current locks 2024-03-26T21:18:08.981 DEBUG:teuthology.nuke:shortname: smithi063 2024-03-26T21:18:08.981 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-03-26T21:18:09.007 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi063.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/gabrioux-2024-03-26_20:31:35-orch:cephadm-wip-guits-testing-2024-03-26-0925-distro-default-smithi/7624079', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-03-26 20:33:58.548966', 'locked_by': 'scheduled_gabrioux@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBC9Hv14bD1Z0Q7RqOa2RYZQs2kYcksnP86pj5bqMVjbFB2k7PsJJwRxCDYCetJe6lOxb1+jyC8DTngpKTqvZILM='} 2024-03-26T21:18:09.010 DEBUG:teuthology.nuke:shortname: smithi174 2024-03-26T21:18:09.010 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-03-26T21:18:09.029 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi174.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/gabrioux-2024-03-26_20:31:35-orch:cephadm-wip-guits-testing-2024-03-26-0925-distro-default-smithi/7624079', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-03-26 20:33:58.550526', 'locked_by': 'scheduled_gabrioux@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBHAyroYl6r4dC7FKie8aWY8wnEkQIOSzm4Ncr41IYmxEaxKxyLwCpHcvF8/9ahDJrrhVKyEhgnaphxd67ZrFGgg='} 2024-03-26T21:18:09.062 INFO:teuthology.orchestra.console.smithi063:Power off 2024-03-26T21:18:09.063 DEBUG:teuthology.orchestra.console.smithi063:pexpect command: ipmitool -H smithi063.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-03-26T21:18:09.086 DEBUG:teuthology.orchestra.console.smithi063:power off output: Chassis Power Control: Down/Off 2024-03-26T21:18:09.087 DEBUG:teuthology.orchestra.console.smithi063:pexpect command: ipmitool -H smithi063.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-03-26T21:18:09.099 INFO:teuthology.orchestra.console.smithi174:Power off 2024-03-26T21:18:09.100 DEBUG:teuthology.orchestra.console.smithi174:pexpect command: ipmitool -H smithi174.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-03-26T21:18:09.109 DEBUG:teuthology.orchestra.console.smithi063:check power output: Chassis Power is on 2024-03-26T21:18:09.122 DEBUG:teuthology.orchestra.console.smithi174:power off output: Chassis Power Control: Down/Off 2024-03-26T21:18:09.122 DEBUG:teuthology.orchestra.console.smithi174:pexpect command: ipmitool -H smithi174.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-03-26T21:18:09.146 DEBUG:teuthology.orchestra.console.smithi174:check power output: Chassis Power is on 2024-03-26T21:18:13.110 DEBUG:teuthology.orchestra.console.smithi063:pexpect command: ipmitool -H smithi063.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-03-26T21:18:13.147 DEBUG:teuthology.orchestra.console.smithi174:pexpect command: ipmitool -H smithi174.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-03-26T21:18:13.220 DEBUG:teuthology.orchestra.console.smithi063:check power output: Chassis Power is on 2024-03-26T21:18:13.258 DEBUG:teuthology.orchestra.console.smithi174:check power output: Chassis Power is on 2024-03-26T21:18:17.224 DEBUG:teuthology.orchestra.console.smithi063:pexpect command: ipmitool -H smithi063.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-03-26T21:18:17.258 DEBUG:teuthology.orchestra.console.smithi174:pexpect command: ipmitool -H smithi174.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-03-26T21:18:17.334 DEBUG:teuthology.orchestra.console.smithi063:check power output: Chassis Power is off 2024-03-26T21:18:17.368 DEBUG:teuthology.orchestra.console.smithi174:check power output: Chassis Power is on 2024-03-26T21:18:17.436 INFO:teuthology.orchestra.console.smithi063:Power off completed 2024-03-26T21:18:21.370 DEBUG:teuthology.orchestra.console.smithi174:pexpect command: ipmitool -H smithi174.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-03-26T21:18:21.480 DEBUG:teuthology.orchestra.console.smithi174:check power output: Chassis Power is off 2024-03-26T21:18:21.581 INFO:teuthology.orchestra.console.smithi174:Power off completed 2024-03-26T21:18:21.684 INFO:teuthology.run:Summary data: description: orch:cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} duration: 2085.6059432029724 failure_reason: '"2024-03-26T20:53:30.336493+0000 mon.smithi063 (mon.0) 503 : cluster [WRN] Health check failed: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded (PG_DEGRADED)" in cluster log' owner: scheduled_gabrioux@teuthology success: false 2024-03-26T21:18:21.685 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-03-26T21:18:21.766 INFO:teuthology.run:FAIL