2024-09-21T10:21:21.712 INFO:root:teuthology version: 1.2.2.dev2+g3752d38 2024-09-21T10:21:21.735 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-21T10:21:21.802 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/gabrioux-2024-09-21_10:07:29-orch:cephadm-wip-guits-main-2024-09-21-0813-distro-default-smithi/7913804 branch: wip-guits-main-2024-09-21-0813 description: orch:cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} email: gabrioux@ibm.com first_in_suite: false job_id: '7913804' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi name: gabrioux-2024-09-21_10:07:29-orch:cephadm-wip-guits-main-2024-09-21-0813-distro-default-smithi no_nested_subset: false openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 9.stream overrides: admin_socket: branch: wip-guits-main-2024-09-21-0813 ceph: conf: mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - CEPHADM_REFRESH_FAILED log-only-match: - CEPHADM_ sha1: 5c1cdc3c470dae78e91a0ed2bcf13509a046d301 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: 5c1cdc3c470dae78e91a0ed2bcf13509a046d301 selinux: allowlist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: wip-guits-main-2024-09-21-0813 sha1: 5c1cdc3c470dae78e91a0ed2bcf13509a046d301 owner: scheduled_gabrioux@teuthology priority: 70 repo: https://git.ceph.com/ceph-ci.git roles: - - host.a - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - - host.b - osd.4 - osd.5 - osd.6 - osd.7 seed: 3041 sha1: 5c1cdc3c470dae78e91a0ed2bcf13509a046d301 sleep_before_teardown: 0 subset: 1/8 suite: orch:cephadm suite_branch: wip-guits-main-2024-09-21-0813 suite_path: /home/teuthworker/src/git.ceph.com_ceph-c_5c1cdc3c470dae78e91a0ed2bcf13509a046d301/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph-ci.git suite_sha1: 5c1cdc3c470dae78e91a0ed2bcf13509a046d301 targets: smithi123.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBIwfHAyALyPwSKy+ePU50dKu1nj9WV8GNdhSoPhb/68KusfSqn5b8xRjIpzNj41VdRQ6MGhzrVCHjT0BKusoG8A= smithi175.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNUicTsXcPpIDDWh5r5t62zVl0FG69F67leJXJ2geJgWbnuVYEnKqDxXxSKjyTomCJ2O9Mc3vvb+LFJMyoFuS9s= tasks: - cephadm: cephadm_branch: v17.2.0 cephadm_git_url: https://github.com/ceph/ceph image: quay.io/ceph/ceph:v17.2.0 roleless: true - cephadm.shell: host.a: - ceph orch status - ceph orch ps - ceph orch ls - ceph orch host ls - ceph orch device ls - vip.exec: all-hosts: - systemctl stop nfs-server - cephadm.shell: host.a: - ceph fs volume create foofs - cephadm.wait_for_service: service: mds.foofs - cephadm.shell: host.a: - ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2 - ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake - while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done - vip.exec: host.a: - mkdir /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done - echo test > /mnt/foo/testfile - sync - parallel: - upgrade-tasks - workload-tasks - vip.exec: host.a: - umount /mnt/foo - cephadm.shell: host.a: - ceph nfs cluster ls | grep foo - ceph nfs export ls foo --detailed - rados -p .nfs --all ls - - cephadm.shell: host.a: - 'set -ex [[ `ceph config get mgr mgr/cephadm/migration_current` -gt 2 ]] ' teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a timestamp: 2024-09-21_10:07:29 tube: smithi upgrade-tasks: sequential: - cephadm.shell: env: - sha1 host.a: - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph config set global log_to_journald false --force - ceph mgr module enable nfs --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - cephadm.shell: env: - sha1 host.a: - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done - ceph orch ps - ceph versions - echo "wait for servicemap items w/ changing names to refresh" - sleep 60 - ceph orch ps - ceph orch upgrade status - ceph health detail - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 - cephadm.wait_for_service: service: nfs.foo user: gabrioux verbose: true worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2902 workload-tasks: sequential: - exec: host.a: - cd /mnt/foo && dbench 5 -t 600 || true - umount /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo ; do sleep 5 ; done - cd /mnt/foo && dbench 5 -t 5 2024-09-21T10:21:21.802 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph-c_5c1cdc3c470dae78e91a0ed2bcf13509a046d301/qa; will attempt to use it 2024-09-21T10:21:21.802 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph-c_5c1cdc3c470dae78e91a0ed2bcf13509a046d301/qa/tasks 2024-09-21T10:21:21.803 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-09-21T10:21:21.804 INFO:teuthology.task.internal:Checking packages... 2024-09-21T10:21:21.824 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash '5c1cdc3c470dae78e91a0ed2bcf13509a046d301' 2024-09-21T10:21:21.824 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-21T10:21:21.824 INFO:teuthology.packaging:ref: None 2024-09-21T10:21:21.824 INFO:teuthology.packaging:tag: None 2024-09-21T10:21:21.824 INFO:teuthology.packaging:branch: wip-guits-main-2024-09-21-0813 2024-09-21T10:21:21.824 INFO:teuthology.packaging:sha1: 5c1cdc3c470dae78e91a0ed2bcf13509a046d301 2024-09-21T10:21:21.824 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=wip-guits-main-2024-09-21-0813 2024-09-21T10:21:21.971 INFO:teuthology.task.internal:Found packages for ceph version 19.3.0-5103.g5c1cdc3c 2024-09-21T10:21:21.972 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-09-21T10:21:21.978 INFO:teuthology.task.internal:no buildpackages task found 2024-09-21T10:21:21.978 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-09-21T10:21:22.000 INFO:teuthology.task.internal:Saving configuration 2024-09-21T10:21:22.012 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-09-21T10:21:22.018 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-09-21T10:21:22.036 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi123.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/gabrioux-2024-09-21_10:07:29-orch:cephadm-wip-guits-main-2024-09-21-0813-distro-default-smithi/7913804', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-21 10:16:27.354033', 'locked_by': 'scheduled_gabrioux@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBIwfHAyALyPwSKy+ePU50dKu1nj9WV8GNdhSoPhb/68KusfSqn5b8xRjIpzNj41VdRQ6MGhzrVCHjT0BKusoG8A='} 2024-09-21T10:21:22.056 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi175.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/gabrioux-2024-09-21_10:07:29-orch:cephadm-wip-guits-main-2024-09-21-0813-distro-default-smithi/7913804', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-21 10:16:27.355478', 'locked_by': 'scheduled_gabrioux@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNUicTsXcPpIDDWh5r5t62zVl0FG69F67leJXJ2geJgWbnuVYEnKqDxXxSKjyTomCJ2O9Mc3vvb+LFJMyoFuS9s='} 2024-09-21T10:21:22.056 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-09-21T10:21:22.062 INFO:teuthology.task.internal:roles: ubuntu@smithi123.front.sepia.ceph.com - ['host.a', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0'] 2024-09-21T10:21:22.062 INFO:teuthology.task.internal:roles: ubuntu@smithi175.front.sepia.ceph.com - ['host.b', 'osd.4', 'osd.5', 'osd.6', 'osd.7'] 2024-09-21T10:21:22.062 INFO:teuthology.run_tasks:Running task console_log... 2024-09-21T10:21:22.159 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f8168ab0280>, signals=[15]) 2024-09-21T10:21:22.159 INFO:teuthology.run_tasks:Running task internal.connect... 2024-09-21T10:21:22.165 INFO:teuthology.task.internal:Opening connections... 2024-09-21T10:21:22.165 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi123.front.sepia.ceph.com 2024-09-21T10:21:22.166 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi123.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T10:21:22.242 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi175.front.sepia.ceph.com 2024-09-21T10:21:22.242 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi175.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T10:21:22.316 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-09-21T10:21:22.322 DEBUG:teuthology.orchestra.run.smithi123:> uname -m 2024-09-21T10:21:22.340 INFO:teuthology.orchestra.run.smithi123.stdout:x86_64 2024-09-21T10:21:22.341 DEBUG:teuthology.orchestra.run.smithi123:> cat /etc/os-release 2024-09-21T10:21:22.398 INFO:teuthology.orchestra.run.smithi123.stdout:NAME="CentOS Stream" 2024-09-21T10:21:22.398 INFO:teuthology.orchestra.run.smithi123.stdout:VERSION="9" 2024-09-21T10:21:22.398 INFO:teuthology.orchestra.run.smithi123.stdout:ID="centos" 2024-09-21T10:21:22.398 INFO:teuthology.orchestra.run.smithi123.stdout:ID_LIKE="rhel fedora" 2024-09-21T10:21:22.398 INFO:teuthology.orchestra.run.smithi123.stdout:VERSION_ID="9" 2024-09-21T10:21:22.398 INFO:teuthology.orchestra.run.smithi123.stdout:PLATFORM_ID="platform:el9" 2024-09-21T10:21:22.398 INFO:teuthology.orchestra.run.smithi123.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-21T10:21:22.398 INFO:teuthology.orchestra.run.smithi123.stdout:ANSI_COLOR="0;31" 2024-09-21T10:21:22.398 INFO:teuthology.orchestra.run.smithi123.stdout:LOGO="fedora-logo-icon" 2024-09-21T10:21:22.398 INFO:teuthology.orchestra.run.smithi123.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-21T10:21:22.398 INFO:teuthology.orchestra.run.smithi123.stdout:HOME_URL="https://centos.org/" 2024-09-21T10:21:22.399 INFO:teuthology.orchestra.run.smithi123.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-21T10:21:22.399 INFO:teuthology.orchestra.run.smithi123.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-21T10:21:22.399 INFO:teuthology.orchestra.run.smithi123.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-21T10:21:22.399 INFO:teuthology.lock.ops:Updating smithi123.front.sepia.ceph.com on lock server 2024-09-21T10:21:22.421 DEBUG:teuthology.orchestra.run.smithi175:> uname -m 2024-09-21T10:21:22.439 INFO:teuthology.orchestra.run.smithi175.stdout:x86_64 2024-09-21T10:21:22.440 DEBUG:teuthology.orchestra.run.smithi175:> cat /etc/os-release 2024-09-21T10:21:22.497 INFO:teuthology.orchestra.run.smithi175.stdout:NAME="CentOS Stream" 2024-09-21T10:21:22.497 INFO:teuthology.orchestra.run.smithi175.stdout:VERSION="9" 2024-09-21T10:21:22.497 INFO:teuthology.orchestra.run.smithi175.stdout:ID="centos" 2024-09-21T10:21:22.497 INFO:teuthology.orchestra.run.smithi175.stdout:ID_LIKE="rhel fedora" 2024-09-21T10:21:22.497 INFO:teuthology.orchestra.run.smithi175.stdout:VERSION_ID="9" 2024-09-21T10:21:22.497 INFO:teuthology.orchestra.run.smithi175.stdout:PLATFORM_ID="platform:el9" 2024-09-21T10:21:22.497 INFO:teuthology.orchestra.run.smithi175.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-21T10:21:22.497 INFO:teuthology.orchestra.run.smithi175.stdout:ANSI_COLOR="0;31" 2024-09-21T10:21:22.497 INFO:teuthology.orchestra.run.smithi175.stdout:LOGO="fedora-logo-icon" 2024-09-21T10:21:22.498 INFO:teuthology.orchestra.run.smithi175.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-21T10:21:22.498 INFO:teuthology.orchestra.run.smithi175.stdout:HOME_URL="https://centos.org/" 2024-09-21T10:21:22.498 INFO:teuthology.orchestra.run.smithi175.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-21T10:21:22.498 INFO:teuthology.orchestra.run.smithi175.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-21T10:21:22.498 INFO:teuthology.orchestra.run.smithi175.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-21T10:21:22.498 INFO:teuthology.lock.ops:Updating smithi175.front.sepia.ceph.com on lock server 2024-09-21T10:21:22.513 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-09-21T10:21:22.521 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-09-21T10:21:22.532 INFO:teuthology.task.internal:Checking for old test directory... 2024-09-21T10:21:22.532 DEBUG:teuthology.orchestra.run.smithi123:> test '!' -e /home/ubuntu/cephtest 2024-09-21T10:21:22.534 DEBUG:teuthology.orchestra.run.smithi175:> test '!' -e /home/ubuntu/cephtest 2024-09-21T10:21:22.553 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-09-21T10:21:22.559 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-09-21T10:21:22.559 DEBUG:teuthology.orchestra.run.smithi123:> test -z $(ls -A /var/lib/ceph) 2024-09-21T10:21:22.591 DEBUG:teuthology.orchestra.run.smithi175:> test -z $(ls -A /var/lib/ceph) 2024-09-21T10:21:22.688 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-09-21T10:21:22.737 INFO:teuthology.run_tasks:Running task kernel... 2024-09-21T10:21:22.750 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-09-21T10:21:22.750 INFO:teuthology.task.kernel:config {'host.a': {'kdb': 1, 'sha1': 'distro'}, 'host.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-09-21T10:21:22.751 DEBUG:teuthology.orchestra.run.smithi123:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-21T10:21:22.751 DEBUG:teuthology.orchestra.run.smithi175:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-21T10:21:22.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:22.767 DEBUG:teuthology.orchestra.run.smithi175:> uname -r 2024-09-21T10:21:22.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:21:22.768 DEBUG:teuthology.orchestra.run.smithi123:> uname -r 2024-09-21T10:21:22.824 INFO:teuthology.orchestra.run.smithi175.stdout:5.14.0-503.el9.x86_64 2024-09-21T10:21:22.824 INFO:teuthology.task.kernel:Running kernel on smithi175: 5.14.0-503.el9.x86_64 2024-09-21T10:21:22.824 DEBUG:teuthology.orchestra.run.smithi175:> sudo yum install -y kernel 2024-09-21T10:21:22.825 INFO:teuthology.orchestra.run.smithi123.stdout:5.14.0-503.el9.x86_64 2024-09-21T10:21:22.825 INFO:teuthology.task.kernel:Running kernel on smithi123: 5.14.0-503.el9.x86_64 2024-09-21T10:21:22.825 DEBUG:teuthology.orchestra.run.smithi123:> sudo yum install -y kernel 2024-09-21T10:21:26.443 INFO:teuthology.orchestra.run.smithi123.stdout:CentOS Stream 9 - BaseOS 4.3 MB/s | 8.3 MB 00:01 2024-09-21T10:21:27.514 INFO:teuthology.orchestra.run.smithi175.stdout:CentOS Stream 9 - BaseOS 2.8 MB/s | 8.3 MB 00:02 2024-09-21T10:21:29.115 INFO:teuthology.orchestra.run.smithi175.stdout:CentOS Stream 9 - AppStream 37 MB/s | 20 MB 00:00 2024-09-21T10:21:29.494 INFO:teuthology.orchestra.run.smithi123.stdout:CentOS Stream 9 - AppStream 10 MB/s | 20 MB 00:01 2024-09-21T10:21:34.782 INFO:teuthology.orchestra.run.smithi175.stdout:CentOS Stream 9 - CRB 3.4 MB/s | 6.5 MB 00:01 2024-09-21T10:21:35.036 INFO:teuthology.orchestra.run.smithi123.stdout:CentOS Stream 9 - CRB 3.7 MB/s | 6.5 MB 00:01 2024-09-21T10:21:36.649 INFO:teuthology.orchestra.run.smithi175.stdout:CentOS Stream 9 - Extras packages 29 kB/s | 19 kB 00:00 2024-09-21T10:21:37.270 INFO:teuthology.orchestra.run.smithi123.stdout:CentOS Stream 9 - Extras packages 19 kB/s | 19 kB 00:01 2024-09-21T10:21:37.578 INFO:teuthology.orchestra.run.smithi175.stdout:Extra Packages for Enterprise Linux 28 MB/s | 23 MB 00:00 2024-09-21T10:21:38.124 INFO:teuthology.orchestra.run.smithi123.stdout:Extra Packages for Enterprise Linux 31 MB/s | 23 MB 00:00 2024-09-21T10:21:42.951 INFO:teuthology.orchestra.run.smithi175.stdout:lab-extras 36 kB/s | 1.7 kB 00:00 2024-09-21T10:21:43.538 INFO:teuthology.orchestra.run.smithi123.stdout:lab-extras 33 kB/s | 1.7 kB 00:00 2024-09-21T10:21:44.799 INFO:teuthology.orchestra.run.smithi175.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-21T10:21:44.799 INFO:teuthology.orchestra.run.smithi175.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-21T10:21:44.897 INFO:teuthology.orchestra.run.smithi175.stdout:Dependencies resolved. 2024-09-21T10:21:44.906 INFO:teuthology.orchestra.run.smithi175.stdout:================================================================================ 2024-09-21T10:21:44.906 INFO:teuthology.orchestra.run.smithi175.stdout: Package Architecture Version Repository Size 2024-09-21T10:21:44.906 INFO:teuthology.orchestra.run.smithi175.stdout:================================================================================ 2024-09-21T10:21:44.906 INFO:teuthology.orchestra.run.smithi175.stdout:Installing: 2024-09-21T10:21:44.907 INFO:teuthology.orchestra.run.smithi175.stdout: kernel x86_64 5.14.0-508.el9 baseos 36 k 2024-09-21T10:21:44.907 INFO:teuthology.orchestra.run.smithi175.stdout:Installing dependencies: 2024-09-21T10:21:44.907 INFO:teuthology.orchestra.run.smithi175.stdout: kernel-core x86_64 5.14.0-508.el9 baseos 16 M 2024-09-21T10:21:44.907 INFO:teuthology.orchestra.run.smithi175.stdout: kernel-modules x86_64 5.14.0-508.el9 baseos 35 M 2024-09-21T10:21:44.907 INFO:teuthology.orchestra.run.smithi175.stdout: kernel-modules-core x86_64 5.14.0-508.el9 baseos 29 M 2024-09-21T10:21:44.907 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-09-21T10:21:44.907 INFO:teuthology.orchestra.run.smithi175.stdout:Transaction Summary 2024-09-21T10:21:44.907 INFO:teuthology.orchestra.run.smithi175.stdout:================================================================================ 2024-09-21T10:21:44.907 INFO:teuthology.orchestra.run.smithi175.stdout:Install 4 Packages 2024-09-21T10:21:44.907 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-09-21T10:21:44.908 INFO:teuthology.orchestra.run.smithi175.stdout:Total download size: 79 M 2024-09-21T10:21:44.908 INFO:teuthology.orchestra.run.smithi175.stdout:Installed size: 126 M 2024-09-21T10:21:44.908 INFO:teuthology.orchestra.run.smithi175.stdout:Downloading Packages: 2024-09-21T10:21:45.285 INFO:teuthology.orchestra.run.smithi175.stdout:(1/4): kernel-5.14.0-508.el9.x86_64.rpm 293 kB/s | 36 kB 00:00 2024-09-21T10:21:45.445 INFO:teuthology.orchestra.run.smithi123.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-21T10:21:45.445 INFO:teuthology.orchestra.run.smithi123.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-21T10:21:45.530 INFO:teuthology.orchestra.run.smithi123.stdout:Dependencies resolved. 2024-09-21T10:21:45.535 INFO:teuthology.orchestra.run.smithi123.stdout:================================================================================ 2024-09-21T10:21:45.535 INFO:teuthology.orchestra.run.smithi123.stdout: Package Architecture Version Repository Size 2024-09-21T10:21:45.535 INFO:teuthology.orchestra.run.smithi123.stdout:================================================================================ 2024-09-21T10:21:45.535 INFO:teuthology.orchestra.run.smithi123.stdout:Installing: 2024-09-21T10:21:45.536 INFO:teuthology.orchestra.run.smithi123.stdout: kernel x86_64 5.14.0-508.el9 baseos 36 k 2024-09-21T10:21:45.536 INFO:teuthology.orchestra.run.smithi123.stdout:Installing dependencies: 2024-09-21T10:21:45.536 INFO:teuthology.orchestra.run.smithi123.stdout: kernel-core x86_64 5.14.0-508.el9 baseos 16 M 2024-09-21T10:21:45.536 INFO:teuthology.orchestra.run.smithi123.stdout: kernel-modules x86_64 5.14.0-508.el9 baseos 35 M 2024-09-21T10:21:45.536 INFO:teuthology.orchestra.run.smithi123.stdout: kernel-modules-core x86_64 5.14.0-508.el9 baseos 29 M 2024-09-21T10:21:45.536 INFO:teuthology.orchestra.run.smithi123.stdout: 2024-09-21T10:21:45.536 INFO:teuthology.orchestra.run.smithi123.stdout:Transaction Summary 2024-09-21T10:21:45.536 INFO:teuthology.orchestra.run.smithi123.stdout:================================================================================ 2024-09-21T10:21:45.536 INFO:teuthology.orchestra.run.smithi123.stdout:Install 4 Packages 2024-09-21T10:21:45.536 INFO:teuthology.orchestra.run.smithi123.stdout: 2024-09-21T10:21:45.537 INFO:teuthology.orchestra.run.smithi123.stdout:Total download size: 79 M 2024-09-21T10:21:45.537 INFO:teuthology.orchestra.run.smithi123.stdout:Installed size: 126 M 2024-09-21T10:21:45.537 INFO:teuthology.orchestra.run.smithi123.stdout:Downloading Packages: 2024-09-21T10:21:45.748 INFO:teuthology.orchestra.run.smithi123.stdout:(1/4): kernel-5.14.0-508.el9.x86_64.rpm 353 kB/s | 36 kB 00:00 2024-09-21T10:21:46.123 INFO:teuthology.orchestra.run.smithi123.stdout:(2/4): kernel-core-5.14.0-508.el9.x86_64.rpm 33 MB/s | 16 MB 00:00 2024-09-21T10:21:46.507 INFO:teuthology.orchestra.run.smithi123.stdout:(3/4): kernel-modules-core-5.14.0-508.el9.x86_6 38 MB/s | 29 MB 00:00 2024-09-21T10:21:46.824 INFO:teuthology.orchestra.run.smithi123.stdout:(4/4): kernel-modules-5.14.0-508.el9.x86_64.rpm 29 MB/s | 35 MB 00:01 2024-09-21T10:21:46.824 INFO:teuthology.orchestra.run.smithi123.stdout:-------------------------------------------------------------------------------- 2024-09-21T10:21:46.824 INFO:teuthology.orchestra.run.smithi123.stdout:Total 61 MB/s | 79 MB 00:01 2024-09-21T10:21:47.503 INFO:teuthology.orchestra.run.smithi123.stdout:Running transaction check 2024-09-21T10:21:47.863 INFO:teuthology.orchestra.run.smithi123.stdout:Transaction check succeeded. 2024-09-21T10:21:47.863 INFO:teuthology.orchestra.run.smithi123.stdout:Running transaction test 2024-09-21T10:21:48.826 INFO:teuthology.orchestra.run.smithi123.stdout:Transaction test succeeded. 2024-09-21T10:21:48.827 INFO:teuthology.orchestra.run.smithi123.stdout:Running transaction 2024-09-21T10:21:51.252 INFO:teuthology.orchestra.run.smithi123.stdout: Preparing : 1/1 2024-09-21T10:21:51.906 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : kernel-modules-core-5.14.0-508.el9.x86_64 1/4 2024-09-21T10:21:51.982 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-21T10:21:52.135 INFO:teuthology.orchestra.run.smithi175.stdout:(2/4): kernel-core-5.14.0-508.el9.x86_64.rpm 2.2 MB/s | 16 MB 00:06 2024-09-21T10:21:54.739 INFO:teuthology.orchestra.run.smithi123.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-21T10:21:54.869 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-21T10:21:58.500 INFO:teuthology.orchestra.run.smithi175.stdout:(3/4): kernel-modules-core-5.14.0-508.el9.x86_6 2.2 MB/s | 29 MB 00:13 2024-09-21T10:21:59.756 INFO:teuthology.orchestra.run.smithi123.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-21T10:21:59.802 INFO:teuthology.orchestra.run.smithi123.stdout: Installing : kernel-5.14.0-508.el9.x86_64 4/4 2024-09-21T10:22:01.142 INFO:teuthology.orchestra.run.smithi175.stdout:(4/4): kernel-modules-5.14.0-508.el9.x86_64.rpm 2.2 MB/s | 35 MB 00:15 2024-09-21T10:22:01.142 INFO:teuthology.orchestra.run.smithi175.stdout:-------------------------------------------------------------------------------- 2024-09-21T10:22:01.142 INFO:teuthology.orchestra.run.smithi175.stdout:Total 4.9 MB/s | 79 MB 00:16 2024-09-21T10:22:01.827 INFO:teuthology.orchestra.run.smithi175.stdout:Running transaction check 2024-09-21T10:22:02.173 INFO:teuthology.orchestra.run.smithi175.stdout:Transaction check succeeded. 2024-09-21T10:22:02.174 INFO:teuthology.orchestra.run.smithi175.stdout:Running transaction test 2024-09-21T10:22:03.147 INFO:teuthology.orchestra.run.smithi175.stdout:Transaction test succeeded. 2024-09-21T10:22:03.147 INFO:teuthology.orchestra.run.smithi175.stdout:Running transaction 2024-09-21T10:22:04.668 INFO:teuthology.orchestra.run.smithi123.stdout: Running scriptlet: kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-21T10:22:05.559 INFO:teuthology.orchestra.run.smithi175.stdout: Preparing : 1/1 2024-09-21T10:22:06.185 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : kernel-modules-core-5.14.0-508.el9.x86_64 1/4 2024-09-21T10:22:06.266 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-21T10:22:09.018 INFO:teuthology.orchestra.run.smithi175.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-21T10:22:09.144 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-21T10:22:13.989 INFO:teuthology.orchestra.run.smithi175.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-21T10:22:14.203 INFO:teuthology.orchestra.run.smithi175.stdout: Installing : kernel-5.14.0-508.el9.x86_64 4/4 2024-09-21T10:22:19.079 INFO:teuthology.orchestra.run.smithi175.stdout: Running scriptlet: kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-21T10:22:42.975 INFO:teuthology.orchestra.run.smithi123.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 4/4 2024-09-21T10:22:43.014 INFO:teuthology.orchestra.run.smithi123.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 4/4 2024-09-21T10:22:43.298 INFO:teuthology.orchestra.run.smithi123.stdout: Running scriptlet: kernel-5.14.0-508.el9.x86_64 4/4 2024-09-21T10:22:43.298 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : kernel-5.14.0-508.el9.x86_64 1/4 2024-09-21T10:22:43.298 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-21T10:22:43.298 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-21T10:22:44.122 INFO:teuthology.orchestra.run.smithi123.stdout: Verifying : kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-21T10:22:44.122 INFO:teuthology.orchestra.run.smithi123.stdout: 2024-09-21T10:22:44.122 INFO:teuthology.orchestra.run.smithi123.stdout:Installed: 2024-09-21T10:22:44.122 INFO:teuthology.orchestra.run.smithi123.stdout: kernel-5.14.0-508.el9.x86_64 2024-09-21T10:22:44.122 INFO:teuthology.orchestra.run.smithi123.stdout: kernel-core-5.14.0-508.el9.x86_64 2024-09-21T10:22:44.122 INFO:teuthology.orchestra.run.smithi123.stdout: kernel-modules-5.14.0-508.el9.x86_64 2024-09-21T10:22:44.122 INFO:teuthology.orchestra.run.smithi123.stdout: kernel-modules-core-5.14.0-508.el9.x86_64 2024-09-21T10:22:44.122 INFO:teuthology.orchestra.run.smithi123.stdout: 2024-09-21T10:22:44.122 INFO:teuthology.orchestra.run.smithi123.stdout:Complete! 2024-09-21T10:22:44.487 DEBUG:teuthology.orchestra.run.smithi123:> rpm -q kernel | sort -rV | head -n 1 2024-09-21T10:22:44.574 INFO:teuthology.orchestra.run.smithi123.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-21T10:22:44.575 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-21T10:22:44.575 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-508.el9.x86_64 2024-09-21T10:22:44.575 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-21T10:22:44.575 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-21T10:22:44.575 INFO:teuthology.task.kernel:Installing distro kernel on host.a... 2024-09-21T10:22:44.575 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi123.front.sepia.ceph.com, path=None, version=distro) 2024-09-21T10:22:44.575 DEBUG:teuthology.orchestra.run.smithi123:> rpm -q kernel | sort -rV | head -n 1 2024-09-21T10:22:44.633 INFO:teuthology.orchestra.run.smithi123.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-21T10:22:44.633 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-21T10:22:44.634 DEBUG:teuthology.orchestra.run.smithi123:> sudo rpm -qi grub2-tools 2024-09-21T10:22:44.671 INFO:teuthology.orchestra.run.smithi123.stdout:Name : grub2-tools 2024-09-21T10:22:44.671 INFO:teuthology.orchestra.run.smithi123.stdout:Epoch : 1 2024-09-21T10:22:44.671 INFO:teuthology.orchestra.run.smithi123.stdout:Version : 2.06 2024-09-21T10:22:44.671 INFO:teuthology.orchestra.run.smithi123.stdout:Release : 82.el9 2024-09-21T10:22:44.671 INFO:teuthology.orchestra.run.smithi123.stdout:Architecture: x86_64 2024-09-21T10:22:44.671 INFO:teuthology.orchestra.run.smithi123.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-21T10:22:44.671 INFO:teuthology.orchestra.run.smithi123.stdout:Group : Unspecified 2024-09-21T10:22:44.671 INFO:teuthology.orchestra.run.smithi123.stdout:Size : 8274670 2024-09-21T10:22:44.671 INFO:teuthology.orchestra.run.smithi123.stdout:License : GPLv3+ 2024-09-21T10:22:44.671 INFO:teuthology.orchestra.run.smithi123.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-21T10:22:44.671 INFO:teuthology.orchestra.run.smithi123.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-21T10:22:44.671 INFO:teuthology.orchestra.run.smithi123.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-21T10:22:44.672 INFO:teuthology.orchestra.run.smithi123.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-21T10:22:44.672 INFO:teuthology.orchestra.run.smithi123.stdout:Packager : builder@centos.org 2024-09-21T10:22:44.672 INFO:teuthology.orchestra.run.smithi123.stdout:Vendor : CentOS 2024-09-21T10:22:44.672 INFO:teuthology.orchestra.run.smithi123.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-21T10:22:44.672 INFO:teuthology.orchestra.run.smithi123.stdout:Summary : Support tools for GRUB. 2024-09-21T10:22:44.672 INFO:teuthology.orchestra.run.smithi123.stdout:Description : 2024-09-21T10:22:44.672 INFO:teuthology.orchestra.run.smithi123.stdout: 2024-09-21T10:22:44.672 INFO:teuthology.orchestra.run.smithi123.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-21T10:22:44.672 INFO:teuthology.orchestra.run.smithi123.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-21T10:22:44.672 INFO:teuthology.orchestra.run.smithi123.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-21T10:22:44.672 INFO:teuthology.orchestra.run.smithi123.stdout:hardware devices. 2024-09-21T10:22:44.672 INFO:teuthology.orchestra.run.smithi123.stdout: 2024-09-21T10:22:44.673 INFO:teuthology.orchestra.run.smithi123.stdout:This subpackage provides tools for support of all platforms. 2024-09-21T10:22:44.673 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-21T10:22:44.673 INFO:teuthology.task.kernel:Updating grub on smithi123 to boot 5.14.0-508.el9.x86_64 2024-09-21T10:22:44.673 DEBUG:teuthology.orchestra.run.smithi123:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-21T10:22:45.324 INFO:teuthology.orchestra.run.smithi123.stderr:Generating grub configuration file ... 2024-09-21T10:22:46.644 INFO:teuthology.orchestra.run.smithi123.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-21T10:22:46.669 INFO:teuthology.orchestra.run.smithi123.stderr:done 2024-09-21T10:22:46.671 DEBUG:teuthology.orchestra.run.smithi123:> mktemp 2024-09-21T10:22:46.687 INFO:teuthology.orchestra.run.smithi123.stdout:/tmp/tmp.dshbqGcQs7 2024-09-21T10:22:46.687 DEBUG:teuthology.orchestra.run.smithi123:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.dshbqGcQs7 2024-09-21T10:22:46.754 DEBUG:teuthology.orchestra.run.smithi123:> sudo chmod 0666 /tmp/tmp.dshbqGcQs7 2024-09-21T10:22:46.898 DEBUG:teuthology.orchestra.remote:smithi123:/tmp/tmp.dshbqGcQs7 is 6KB 2024-09-21T10:22:46.946 DEBUG:teuthology.orchestra.run.smithi123:> rm -fr /tmp/tmp.dshbqGcQs7 2024-09-21T10:22:46.960 DEBUG:teuthology.orchestra.run.smithi123:> sudo /bin/ls /boot/loader/entries || true 2024-09-21T10:22:47.022 INFO:teuthology.orchestra.run.smithi123.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-21T10:22:47.022 INFO:teuthology.orchestra.run.smithi123.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-21T10:22:47.023 INFO:teuthology.orchestra.run.smithi123.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-21T10:22:47.023 INFO:teuthology.orchestra.run.smithi123.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64.conf 2024-09-21T10:22:47.024 DEBUG:teuthology.orchestra.run.smithi123:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64 2024-09-21T10:22:47.185 DEBUG:teuthology.orchestra.run.smithi123:> sudo shutdown -r now 2024-09-21T10:22:57.007 INFO:teuthology.orchestra.run.smithi175.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 4/4 2024-09-21T10:22:57.047 INFO:teuthology.orchestra.run.smithi175.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 4/4 2024-09-21T10:22:57.460 INFO:teuthology.orchestra.run.smithi175.stdout: Running scriptlet: kernel-5.14.0-508.el9.x86_64 4/4 2024-09-21T10:22:57.460 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : kernel-5.14.0-508.el9.x86_64 1/4 2024-09-21T10:22:57.460 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-21T10:22:57.460 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-21T10:22:58.300 INFO:teuthology.orchestra.run.smithi175.stdout: Verifying : kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-21T10:22:58.300 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-09-21T10:22:58.300 INFO:teuthology.orchestra.run.smithi175.stdout:Installed: 2024-09-21T10:22:58.300 INFO:teuthology.orchestra.run.smithi175.stdout: kernel-5.14.0-508.el9.x86_64 2024-09-21T10:22:58.300 INFO:teuthology.orchestra.run.smithi175.stdout: kernel-core-5.14.0-508.el9.x86_64 2024-09-21T10:22:58.300 INFO:teuthology.orchestra.run.smithi175.stdout: kernel-modules-5.14.0-508.el9.x86_64 2024-09-21T10:22:58.300 INFO:teuthology.orchestra.run.smithi175.stdout: kernel-modules-core-5.14.0-508.el9.x86_64 2024-09-21T10:22:58.300 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-09-21T10:22:58.300 INFO:teuthology.orchestra.run.smithi175.stdout:Complete! 2024-09-21T10:22:58.660 DEBUG:teuthology.orchestra.run.smithi175:> rpm -q kernel | sort -rV | head -n 1 2024-09-21T10:22:58.742 INFO:teuthology.orchestra.run.smithi175.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-21T10:22:58.743 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-21T10:22:58.743 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-508.el9.x86_64 2024-09-21T10:22:58.743 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-21T10:22:58.743 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-21T10:22:58.743 INFO:teuthology.task.kernel:Installing distro kernel on host.b... 2024-09-21T10:22:58.743 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi175.front.sepia.ceph.com, path=None, version=distro) 2024-09-21T10:22:58.743 DEBUG:teuthology.orchestra.run.smithi175:> rpm -q kernel | sort -rV | head -n 1 2024-09-21T10:22:58.802 INFO:teuthology.orchestra.run.smithi175.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-21T10:22:58.803 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-21T10:22:58.803 DEBUG:teuthology.orchestra.run.smithi175:> sudo rpm -qi grub2-tools 2024-09-21T10:22:58.845 INFO:teuthology.orchestra.run.smithi175.stdout:Name : grub2-tools 2024-09-21T10:22:58.845 INFO:teuthology.orchestra.run.smithi175.stdout:Epoch : 1 2024-09-21T10:22:58.845 INFO:teuthology.orchestra.run.smithi175.stdout:Version : 2.06 2024-09-21T10:22:58.846 INFO:teuthology.orchestra.run.smithi175.stdout:Release : 82.el9 2024-09-21T10:22:58.846 INFO:teuthology.orchestra.run.smithi175.stdout:Architecture: x86_64 2024-09-21T10:22:58.846 INFO:teuthology.orchestra.run.smithi175.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-21T10:22:58.846 INFO:teuthology.orchestra.run.smithi175.stdout:Group : Unspecified 2024-09-21T10:22:58.846 INFO:teuthology.orchestra.run.smithi175.stdout:Size : 8274670 2024-09-21T10:22:58.846 INFO:teuthology.orchestra.run.smithi175.stdout:License : GPLv3+ 2024-09-21T10:22:58.846 INFO:teuthology.orchestra.run.smithi175.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-21T10:22:58.846 INFO:teuthology.orchestra.run.smithi175.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-21T10:22:58.846 INFO:teuthology.orchestra.run.smithi175.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-21T10:22:58.846 INFO:teuthology.orchestra.run.smithi175.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-21T10:22:58.846 INFO:teuthology.orchestra.run.smithi175.stdout:Packager : builder@centos.org 2024-09-21T10:22:58.846 INFO:teuthology.orchestra.run.smithi175.stdout:Vendor : CentOS 2024-09-21T10:22:58.847 INFO:teuthology.orchestra.run.smithi175.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-21T10:22:58.847 INFO:teuthology.orchestra.run.smithi175.stdout:Summary : Support tools for GRUB. 2024-09-21T10:22:58.847 INFO:teuthology.orchestra.run.smithi175.stdout:Description : 2024-09-21T10:22:58.847 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-09-21T10:22:58.847 INFO:teuthology.orchestra.run.smithi175.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-21T10:22:58.847 INFO:teuthology.orchestra.run.smithi175.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-21T10:22:58.847 INFO:teuthology.orchestra.run.smithi175.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-21T10:22:58.847 INFO:teuthology.orchestra.run.smithi175.stdout:hardware devices. 2024-09-21T10:22:58.847 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-09-21T10:22:58.847 INFO:teuthology.orchestra.run.smithi175.stdout:This subpackage provides tools for support of all platforms. 2024-09-21T10:22:58.848 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-21T10:22:58.848 INFO:teuthology.task.kernel:Updating grub on smithi175 to boot 5.14.0-508.el9.x86_64 2024-09-21T10:22:58.848 DEBUG:teuthology.orchestra.run.smithi175:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-21T10:22:59.488 INFO:teuthology.orchestra.run.smithi175.stderr:Generating grub configuration file ... 2024-09-21T10:23:00.867 INFO:teuthology.orchestra.run.smithi175.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-21T10:23:00.905 INFO:teuthology.orchestra.run.smithi175.stderr:done 2024-09-21T10:23:00.908 DEBUG:teuthology.orchestra.run.smithi175:> mktemp 2024-09-21T10:23:00.924 INFO:teuthology.orchestra.run.smithi175.stdout:/tmp/tmp.1IU1gGK9ox 2024-09-21T10:23:00.924 DEBUG:teuthology.orchestra.run.smithi175:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.1IU1gGK9ox 2024-09-21T10:23:01.001 DEBUG:teuthology.orchestra.run.smithi175:> sudo chmod 0666 /tmp/tmp.1IU1gGK9ox 2024-09-21T10:23:01.134 DEBUG:teuthology.orchestra.remote:smithi175:/tmp/tmp.1IU1gGK9ox is 6KB 2024-09-21T10:23:01.183 DEBUG:teuthology.orchestra.run.smithi175:> rm -fr /tmp/tmp.1IU1gGK9ox 2024-09-21T10:23:01.198 DEBUG:teuthology.orchestra.run.smithi175:> sudo /bin/ls /boot/loader/entries || true 2024-09-21T10:23:01.260 INFO:teuthology.orchestra.run.smithi175.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-21T10:23:01.261 INFO:teuthology.orchestra.run.smithi175.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-21T10:23:01.261 INFO:teuthology.orchestra.run.smithi175.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-21T10:23:01.261 INFO:teuthology.orchestra.run.smithi175.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64.conf 2024-09-21T10:23:01.262 DEBUG:teuthology.orchestra.run.smithi175:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64 2024-09-21T10:23:01.411 DEBUG:teuthology.orchestra.run.smithi175:> sudo shutdown -r now 2024-09-21T10:23:17.204 INFO:teuthology.task.kernel:Checking client host.a for new kernel version... 2024-09-21T10:23:17.204 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi123.front.sepia.ceph.com' 2024-09-21T10:23:17.204 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi123.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T10:23:31.428 INFO:teuthology.task.kernel:Checking client host.b for new kernel version... 2024-09-21T10:23:31.429 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi175.front.sepia.ceph.com' 2024-09-21T10:23:31.429 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi175.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T10:23:35.663 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.123 2024-09-21T10:23:44.670 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi123.front.sepia.ceph.com' 2024-09-21T10:23:44.671 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi123.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T10:23:47.727 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.123 2024-09-21T10:23:59.728 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi123.front.sepia.ceph.com' 2024-09-21T10:23:59.729 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi123.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T10:24:31.431 DEBUG:teuthology.orchestra.remote:timed out 2024-09-21T10:24:31.599 DEBUG:teuthology.orchestra.run.smithi123:> true 2024-09-21T10:24:32.155 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi123.front.sepia.ceph.com' 2024-09-21T10:24:32.155 INFO:teuthology.task.kernel:Checking kernel version of host.a, want "5.14.0-508.el9.x86_64"... 2024-09-21T10:24:32.155 DEBUG:teuthology.orchestra.run.smithi123:> uname -r 2024-09-21T10:24:32.209 INFO:teuthology.orchestra.run.smithi123.stdout:5.14.0-508.el9.x86_64 2024-09-21T10:24:32.210 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-508.el9.x86_64 vs 5.14.0-508.el9.x86_64 2024-09-21T10:24:32.210 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-21T10:24:32.210 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-21T10:24:33.211 INFO:teuthology.task.kernel:Enabling kdb on host.a... 2024-09-21T10:24:33.211 DEBUG:teuthology.orchestra.run.smithi123:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-21T10:24:33.609 INFO:teuthology.orchestra.run.smithi123.stdout:ttyS1 2024-09-21T10:24:33.639 DEBUG:teuthology.parallel:result is None 2024-09-21T10:24:40.432 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi175.front.sepia.ceph.com' 2024-09-21T10:24:40.433 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi175.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T10:24:40.822 DEBUG:teuthology.orchestra.run.smithi175:> true 2024-09-21T10:24:41.231 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi175.front.sepia.ceph.com' 2024-09-21T10:24:41.231 INFO:teuthology.task.kernel:Checking kernel version of host.b, want "5.14.0-508.el9.x86_64"... 2024-09-21T10:24:41.231 DEBUG:teuthology.orchestra.run.smithi175:> uname -r 2024-09-21T10:24:41.246 INFO:teuthology.orchestra.run.smithi175.stdout:5.14.0-508.el9.x86_64 2024-09-21T10:24:41.246 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-508.el9.x86_64 vs 5.14.0-508.el9.x86_64 2024-09-21T10:24:41.246 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-21T10:24:41.246 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-21T10:24:42.247 INFO:teuthology.task.kernel:Enabling kdb on host.b... 2024-09-21T10:24:42.247 DEBUG:teuthology.orchestra.run.smithi175:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-21T10:24:42.358 INFO:teuthology.orchestra.run.smithi175.stdout:ttyS1 2024-09-21T10:24:42.384 DEBUG:teuthology.parallel:result is None 2024-09-21T10:24:42.385 INFO:teuthology.run_tasks:Running task internal.base... 2024-09-21T10:24:42.403 INFO:teuthology.task.internal:Creating test directory... 2024-09-21T10:24:42.403 DEBUG:teuthology.orchestra.run.smithi123:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-21T10:24:42.406 DEBUG:teuthology.orchestra.run.smithi175:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-21T10:24:42.421 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-09-21T10:24:42.433 INFO:teuthology.run_tasks:Running task internal.archive... 2024-09-21T10:24:42.445 INFO:teuthology.task.internal:Creating archive directory... 2024-09-21T10:24:42.445 DEBUG:teuthology.orchestra.run.smithi123:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-21T10:24:42.464 DEBUG:teuthology.orchestra.run.smithi175:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-21T10:24:42.553 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-09-21T10:24:42.562 INFO:teuthology.task.internal:Enabling coredump saving... 2024-09-21T10:24:42.562 DEBUG:teuthology.orchestra.run.smithi123:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-21T10:24:42.564 DEBUG:teuthology.orchestra.run.smithi175:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-21T10:24:42.594 INFO:teuthology.orchestra.run.smithi175.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-21T10:24:42.599 INFO:teuthology.orchestra.run.smithi123.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-21T10:24:42.605 INFO:teuthology.orchestra.run.smithi175.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-21T10:24:42.612 INFO:teuthology.orchestra.run.smithi123.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-21T10:24:42.613 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-09-21T10:24:42.620 INFO:teuthology.task.internal:Configuring sudo... 2024-09-21T10:24:42.621 DEBUG:teuthology.orchestra.run.smithi123:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-21T10:24:42.656 DEBUG:teuthology.orchestra.run.smithi175:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-21T10:24:42.683 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-09-21T10:24:42.696 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-09-21T10:24:42.696 DEBUG:teuthology.orchestra.run.smithi123:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-21T10:24:42.727 DEBUG:teuthology.orchestra.run.smithi175:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-21T10:24:42.743 DEBUG:teuthology.orchestra.run.smithi123:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-21T10:24:42.826 DEBUG:teuthology.orchestra.run.smithi123:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-21T10:24:42.882 DEBUG:teuthology.orchestra.run.smithi123:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-21T10:24:42.968 DEBUG:teuthology.orchestra.run.smithi123:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-21T10:24:43.008 DEBUG:teuthology.orchestra.run.smithi123:> set -ex 2024-09-21T10:24:43.008 DEBUG:teuthology.orchestra.run.smithi123:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-21T10:24:43.075 DEBUG:teuthology.orchestra.run.smithi175:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-21T10:24:43.114 DEBUG:teuthology.orchestra.run.smithi175:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-21T10:24:43.166 DEBUG:teuthology.orchestra.run.smithi175:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-21T10:24:43.249 DEBUG:teuthology.orchestra.run.smithi175:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-21T10:24:43.287 DEBUG:teuthology.orchestra.run.smithi175:> set -ex 2024-09-21T10:24:43.287 DEBUG:teuthology.orchestra.run.smithi175:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-21T10:24:43.350 DEBUG:teuthology.orchestra.run.smithi123:> sudo service rsyslog restart 2024-09-21T10:24:43.353 DEBUG:teuthology.orchestra.run.smithi175:> sudo service rsyslog restart 2024-09-21T10:24:43.395 INFO:teuthology.orchestra.run.smithi123.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-21T10:24:43.442 INFO:teuthology.orchestra.run.smithi175.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-21T10:24:43.945 INFO:teuthology.run_tasks:Running task internal.timer... 2024-09-21T10:24:43.954 INFO:teuthology.task.internal:Starting timer... 2024-09-21T10:24:43.954 INFO:teuthology.run_tasks:Running task pcp... 2024-09-21T10:24:43.970 INFO:teuthology.run_tasks:Running task selinux... 2024-09-21T10:24:44.006 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0']} 2024-09-21T10:24:44.007 DEBUG:teuthology.orchestra.run.smithi123:> sudo service auditd rotate 2024-09-21T10:24:44.065 INFO:teuthology.orchestra.run.smithi123.stdout:Rotating logs: 2024-09-21T10:24:44.067 DEBUG:teuthology.orchestra.run.smithi175:> sudo service auditd rotate 2024-09-21T10:24:44.151 INFO:teuthology.orchestra.run.smithi175.stdout:Rotating logs: 2024-09-21T10:24:44.153 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-09-21T10:24:44.153 DEBUG:teuthology.orchestra.run.smithi123:> /usr/sbin/getenforce 2024-09-21T10:24:44.177 INFO:teuthology.orchestra.run.smithi123.stdout:Permissive 2024-09-21T10:24:44.178 DEBUG:teuthology.orchestra.run.smithi175:> /usr/sbin/getenforce 2024-09-21T10:24:44.203 INFO:teuthology.orchestra.run.smithi175.stdout:Permissive 2024-09-21T10:24:44.203 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi123.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi175.front.sepia.ceph.com': 'permissive'} 2024-09-21T10:24:44.204 DEBUG:teuthology.orchestra.run.smithi123:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-21T10:24:44.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:44.250 DEBUG:teuthology.orchestra.run.smithi175:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-21T10:24:44.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:24:44.276 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-09-21T10:24:44.276 DEBUG:teuthology.orchestra.run.smithi123:> sudo /usr/sbin/setenforce permissive 2024-09-21T10:24:44.322 DEBUG:teuthology.orchestra.run.smithi175:> sudo /usr/sbin/setenforce permissive 2024-09-21T10:24:44.359 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-09-21T10:24:44.378 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2024-09-21T10:24:44.381 INFO:teuthology.repo_utils:Fetching git.ceph.com_ceph-cm-ansible_main from origin 2024-09-21T10:24:44.488 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-09-21T10:24:44.498 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-09-21T10:24:44.499 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi123.front.sepia.ceph.com,smithi175.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-09-21T10:28:26.126 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi123.front.sepia.ceph.com'), Remote(name='ubuntu@smithi175.front.sepia.ceph.com')] 2024-09-21T10:28:26.126 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi123.front.sepia.ceph.com' 2024-09-21T10:28:26.127 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi123.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T10:28:26.204 DEBUG:teuthology.orchestra.run.smithi123:> true 2024-09-21T10:28:26.285 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi123.front.sepia.ceph.com' 2024-09-21T10:28:26.286 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi175.front.sepia.ceph.com' 2024-09-21T10:28:26.286 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi175.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T10:28:26.360 DEBUG:teuthology.orchestra.run.smithi175:> true 2024-09-21T10:28:26.397 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi175.front.sepia.ceph.com' 2024-09-21T10:28:26.397 INFO:teuthology.run_tasks:Running task clock... 2024-09-21T10:28:26.413 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-09-21T10:28:26.413 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-21T10:28:26.413 DEBUG:teuthology.orchestra.run.smithi123:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-21T10:28:26.415 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-21T10:28:26.415 DEBUG:teuthology.orchestra.run.smithi175:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-21T10:28:26.442 INFO:teuthology.orchestra.run.smithi123.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-21T10:28:26.457 INFO:teuthology.orchestra.run.smithi123.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-21T10:28:26.467 INFO:teuthology.orchestra.run.smithi175.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-21T10:28:26.481 INFO:teuthology.orchestra.run.smithi123.stderr:sudo: ntpd: command not found 2024-09-21T10:28:26.482 INFO:teuthology.orchestra.run.smithi175.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-21T10:28:26.491 INFO:teuthology.orchestra.run.smithi123.stdout:506 Cannot talk to daemon 2024-09-21T10:28:26.505 INFO:teuthology.orchestra.run.smithi123.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-21T10:28:26.507 INFO:teuthology.orchestra.run.smithi175.stderr:sudo: ntpd: command not found 2024-09-21T10:28:26.518 INFO:teuthology.orchestra.run.smithi175.stdout:506 Cannot talk to daemon 2024-09-21T10:28:26.519 INFO:teuthology.orchestra.run.smithi123.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-21T10:28:26.532 INFO:teuthology.orchestra.run.smithi175.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-21T10:28:26.546 INFO:teuthology.orchestra.run.smithi175.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-21T10:28:26.576 INFO:teuthology.orchestra.run.smithi123.stderr:bash: line 1: ntpq: command not found 2024-09-21T10:28:26.580 INFO:teuthology.orchestra.run.smithi123.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-21T10:28:26.580 INFO:teuthology.orchestra.run.smithi123.stdout:=============================================================================== 2024-09-21T10:28:26.580 INFO:teuthology.orchestra.run.smithi123.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T10:28:26.580 INFO:teuthology.orchestra.run.smithi123.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T10:28:26.580 INFO:teuthology.orchestra.run.smithi123.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T10:28:26.580 INFO:teuthology.orchestra.run.smithi123.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T10:28:26.608 INFO:teuthology.orchestra.run.smithi175.stderr:bash: line 1: ntpq: command not found 2024-09-21T10:28:26.611 INFO:teuthology.orchestra.run.smithi175.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-21T10:28:26.611 INFO:teuthology.orchestra.run.smithi175.stdout:=============================================================================== 2024-09-21T10:28:26.611 INFO:teuthology.orchestra.run.smithi175.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T10:28:26.611 INFO:teuthology.orchestra.run.smithi175.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T10:28:26.611 INFO:teuthology.orchestra.run.smithi175.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T10:28:26.611 INFO:teuthology.orchestra.run.smithi175.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T10:28:26.612 INFO:teuthology.run_tasks:Running task cephadm... 2024-09-21T10:28:26.704 INFO:tasks.cephadm:Config: {'cephadm_branch': 'v17.2.0', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'image': 'quay.io/ceph/ceph:v17.2.0', 'roleless': True, 'conf': {'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', 'CEPHADM_REFRESH_FAILED'], 'log-only-match': ['CEPHADM_'], 'sha1': '5c1cdc3c470dae78e91a0ed2bcf13509a046d301'} 2024-09-21T10:28:26.704 INFO:tasks.cephadm:Cluster image is quay.io/ceph/ceph:v17.2.0 2024-09-21T10:28:26.705 INFO:tasks.cephadm:Cluster fsid is 3ccd15e6-7804-11ef-baf6-efdc52797490 2024-09-21T10:28:26.705 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-09-21T10:28:26.705 INFO:tasks.cephadm:No mon roles; fabricating mons 2024-09-21T10:28:26.705 INFO:tasks.cephadm:Monitor IPs: {'mon.smithi123': '172.21.15.123', 'mon.smithi175': '172.21.15.175'} 2024-09-21T10:28:26.705 INFO:tasks.cephadm:Normalizing hostnames... 2024-09-21T10:28:26.706 DEBUG:teuthology.orchestra.run.smithi123:> sudo hostname $(hostname -s) 2024-09-21T10:28:26.731 DEBUG:teuthology.orchestra.run.smithi175:> sudo hostname $(hostname -s) 2024-09-21T10:28:26.758 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref v17.2.0)... 2024-09-21T10:28:26.758 DEBUG:teuthology.orchestra.run.smithi123:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-09-21T10:28:27.861 INFO:teuthology.orchestra.run.smithi123.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Sep 21 10:28 /home/ubuntu/cephtest/cephadm 2024-09-21T10:28:27.862 DEBUG:teuthology.orchestra.run.smithi175:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-09-21T10:28:27.996 INFO:teuthology.orchestra.run.smithi175.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Sep 21 10:28 /home/ubuntu/cephtest/cephadm 2024-09-21T10:28:27.996 DEBUG:teuthology.orchestra.run.smithi123:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-09-21T10:28:28.015 DEBUG:teuthology.orchestra.run.smithi175:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-09-21T10:28:28.042 INFO:tasks.cephadm:Pulling image quay.io/ceph/ceph:v17.2.0 on all hosts... 2024-09-21T10:28:28.042 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-09-21T10:28:28.059 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-09-21T10:28:29.738 INFO:teuthology.orchestra.run.smithi175.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-09-21T10:28:29.802 INFO:teuthology.orchestra.run.smithi123.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-09-21T10:28:41.473 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-09-21T10:28:41.473 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-09-21T10:28:41.473 INFO:teuthology.orchestra.run.smithi123.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-09-21T10:28:41.473 INFO:teuthology.orchestra.run.smithi123.stdout: "repo_digests": [ 2024-09-21T10:28:41.473 INFO:teuthology.orchestra.run.smithi123.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-09-21T10:28:41.473 INFO:teuthology.orchestra.run.smithi123.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-09-21T10:28:41.473 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2024-09-21T10:28:41.473 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-09-21T10:28:42.214 INFO:teuthology.orchestra.run.smithi175.stdout:{ 2024-09-21T10:28:42.214 INFO:teuthology.orchestra.run.smithi175.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-09-21T10:28:42.214 INFO:teuthology.orchestra.run.smithi175.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-09-21T10:28:42.214 INFO:teuthology.orchestra.run.smithi175.stdout: "repo_digests": [ 2024-09-21T10:28:42.214 INFO:teuthology.orchestra.run.smithi175.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-09-21T10:28:42.214 INFO:teuthology.orchestra.run.smithi175.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-09-21T10:28:42.215 INFO:teuthology.orchestra.run.smithi175.stdout: ] 2024-09-21T10:28:42.215 INFO:teuthology.orchestra.run.smithi175.stdout:} 2024-09-21T10:28:42.231 DEBUG:teuthology.orchestra.run.smithi123:> sudo mkdir -p /etc/ceph 2024-09-21T10:28:42.261 DEBUG:teuthology.orchestra.run.smithi175:> sudo mkdir -p /etc/ceph 2024-09-21T10:28:42.288 DEBUG:teuthology.orchestra.run.smithi123:> sudo chmod 777 /etc/ceph 2024-09-21T10:28:42.328 DEBUG:teuthology.orchestra.run.smithi175:> sudo chmod 777 /etc/ceph 2024-09-21T10:28:42.352 INFO:tasks.cephadm:Writing seed config... 2024-09-21T10:28:42.353 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-09-21T10:28:42.353 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-09-21T10:28:42.353 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-09-21T10:28:42.353 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-09-21T10:28:42.353 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-09-21T10:28:42.354 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-09-21T10:28:42.354 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-09-21T10:28:42.354 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-09-21T10:28:42.354 DEBUG:teuthology.orchestra.run.smithi123:> set -ex 2024-09-21T10:28:42.354 DEBUG:teuthology.orchestra.run.smithi123:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-09-21T10:28:42.384 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 3ccd15e6-7804-11ef-baf6-efdc52797490 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-09-21T10:28:42.385 DEBUG:teuthology.orchestra.run.smithi123:mon.smithi123> sudo journalctl -f -n 0 -u ceph-3ccd15e6-7804-11ef-baf6-efdc52797490@mon.smithi123.service 2024-09-21T10:28:42.427 INFO:tasks.cephadm:Bootstrapping... 2024-09-21T10:28:42.427 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 -v bootstrap --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-ip 172.21.15.123 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-09-21T10:28:42.618 INFO:teuthology.orchestra.run.smithi123.stderr:-------------------------------------------------------------------------------- 2024-09-21T10:28:42.618 INFO:teuthology.orchestra.run.smithi123.stderr:cephadm ['--image', 'quay.io/ceph/ceph:v17.2.0', '-v', 'bootstrap', '--fsid', '3ccd15e6-7804-11ef-baf6-efdc52797490', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-ip', '172.21.15.123', '--skip-admin-label'] 2024-09-21T10:28:42.644 INFO:teuthology.orchestra.run.smithi123.stderr:/bin/podman: 5.2.2 2024-09-21T10:28:42.647 INFO:teuthology.orchestra.run.smithi123.stderr:Verifying podman|docker is present... 2024-09-21T10:28:42.668 INFO:teuthology.orchestra.run.smithi123.stderr:/bin/podman: 5.2.2 2024-09-21T10:28:42.670 INFO:teuthology.orchestra.run.smithi123.stderr:Verifying lvm2 is present... 2024-09-21T10:28:42.670 INFO:teuthology.orchestra.run.smithi123.stderr:Verifying time synchronization is in place... 2024-09-21T10:28:42.677 INFO:teuthology.orchestra.run.smithi123.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-09-21T10:28:42.684 INFO:teuthology.orchestra.run.smithi123.stderr:systemctl: inactive 2024-09-21T10:28:42.690 INFO:teuthology.orchestra.run.smithi123.stderr:systemctl: enabled 2024-09-21T10:28:42.696 INFO:teuthology.orchestra.run.smithi123.stderr:systemctl: active 2024-09-21T10:28:42.697 INFO:teuthology.orchestra.run.smithi123.stderr:Unit chronyd.service is enabled and running 2024-09-21T10:28:42.697 INFO:teuthology.orchestra.run.smithi123.stderr:Repeating the final host check... 2024-09-21T10:28:42.717 INFO:teuthology.orchestra.run.smithi123.stderr:/bin/podman: 5.2.2 2024-09-21T10:28:42.719 INFO:teuthology.orchestra.run.smithi123.stderr:podman (/bin/podman) version 5.2.2 is present 2024-09-21T10:28:42.719 INFO:teuthology.orchestra.run.smithi123.stderr:systemctl is present 2024-09-21T10:28:42.719 INFO:teuthology.orchestra.run.smithi123.stderr:lvcreate is present 2024-09-21T10:28:42.725 INFO:teuthology.orchestra.run.smithi123.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-09-21T10:28:42.731 INFO:teuthology.orchestra.run.smithi123.stderr:systemctl: inactive 2024-09-21T10:28:42.737 INFO:teuthology.orchestra.run.smithi123.stderr:systemctl: enabled 2024-09-21T10:28:42.743 INFO:teuthology.orchestra.run.smithi123.stderr:systemctl: active 2024-09-21T10:28:42.744 INFO:teuthology.orchestra.run.smithi123.stderr:Unit chronyd.service is enabled and running 2024-09-21T10:28:42.744 INFO:teuthology.orchestra.run.smithi123.stderr:Host looks OK 2024-09-21T10:28:42.744 INFO:teuthology.orchestra.run.smithi123.stderr:Cluster fsid: 3ccd15e6-7804-11ef-baf6-efdc52797490 2024-09-21T10:28:42.745 INFO:teuthology.orchestra.run.smithi123.stderr:Acquiring lock 139843636774800 on /run/cephadm/3ccd15e6-7804-11ef-baf6-efdc52797490.lock 2024-09-21T10:28:42.745 INFO:teuthology.orchestra.run.smithi123.stderr:Lock 139843636774800 acquired on /run/cephadm/3ccd15e6-7804-11ef-baf6-efdc52797490.lock 2024-09-21T10:28:42.745 INFO:teuthology.orchestra.run.smithi123.stderr:Verifying IP 172.21.15.123 port 3300 ... 2024-09-21T10:28:42.745 INFO:teuthology.orchestra.run.smithi123.stderr:Verifying IP 172.21.15.123 port 6789 ... 2024-09-21T10:28:42.746 INFO:teuthology.orchestra.run.smithi123.stderr:Base mon IP is 172.21.15.123, final addrv is [v2:172.21.15.123:3300,v1:172.21.15.123:6789] 2024-09-21T10:28:42.748 INFO:teuthology.orchestra.run.smithi123.stderr:/sbin/ip: default via 172.21.15.254 dev enp3s0f1 proto dhcp src 172.21.15.123 metric 100 2024-09-21T10:28:42.748 INFO:teuthology.orchestra.run.smithi123.stderr:/sbin/ip: 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.123 metric 100 2024-09-21T10:28:42.751 INFO:teuthology.orchestra.run.smithi123.stderr:/sbin/ip: ::1 dev lo proto kernel metric 256 pref medium 2024-09-21T10:28:42.751 INFO:teuthology.orchestra.run.smithi123.stderr:/sbin/ip: fe80::/64 dev enp3s0f1 proto kernel metric 1024 pref medium 2024-09-21T10:28:42.751 INFO:teuthology.orchestra.run.smithi123.stderr:/sbin/ip: default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 100 pref medium 2024-09-21T10:28:42.753 INFO:teuthology.orchestra.run.smithi123.stderr:/sbin/ip: 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-09-21T10:28:42.753 INFO:teuthology.orchestra.run.smithi123.stderr:/sbin/ip: inet6 ::1/128 scope host 2024-09-21T10:28:42.753 INFO:teuthology.orchestra.run.smithi123.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-09-21T10:28:42.754 INFO:teuthology.orchestra.run.smithi123.stderr:/sbin/ip: 5: enp3s0f1: mtu 1500 state UP qlen 1000 2024-09-21T10:28:42.754 INFO:teuthology.orchestra.run.smithi123.stderr:/sbin/ip: inet6 fe80::ec4:7aff:fe8f:d3f1/64 scope link noprefixroute 2024-09-21T10:28:42.754 INFO:teuthology.orchestra.run.smithi123.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-09-21T10:28:42.754 INFO:teuthology.orchestra.run.smithi123.stderr:Mon IP `172.21.15.123` is in CIDR network `172.21.0.0/20` 2024-09-21T10:28:42.754 INFO:teuthology.orchestra.run.smithi123.stderr:- internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-09-21T10:28:42.755 INFO:teuthology.orchestra.run.smithi123.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-09-21T10:28:42.853 INFO:teuthology.orchestra.run.smithi123.stderr:/bin/podman: Trying to pull quay.io/ceph/ceph:v17.2.0... 2024-09-21T10:28:43.201 INFO:teuthology.orchestra.run.smithi123.stderr:/bin/podman: Getting image source signatures 2024-09-21T10:28:43.201 INFO:teuthology.orchestra.run.smithi123.stderr:/bin/podman: Copying blob sha256:33ca8fff7868c4dc0c11e09bca97c720eb9cfbab7221216754367dd8de70388a 2024-09-21T10:28:43.201 INFO:teuthology.orchestra.run.smithi123.stderr:/bin/podman: Copying blob sha256:89b4a75bc2d8500f15463747507c9623df43886c134463e7f0527e70900e7a7b 2024-09-21T10:28:43.201 INFO:teuthology.orchestra.run.smithi123.stderr:/bin/podman: Copying blob sha256:c32ab78b488d0c72f64eded765c0cf6b5bf2c75dab66cb62a9d367fa6ec42513 2024-09-21T10:28:43.201 INFO:teuthology.orchestra.run.smithi123.stderr:/bin/podman: Copying blob sha256:599d07cb321ff0a3c82224e1138fc685793fa69b93ed5780415751a5f7e4b8c2 2024-09-21T10:28:43.201 INFO:teuthology.orchestra.run.smithi123.stderr:/bin/podman: Copying blob sha256:a70843738bb77e1ab9c1f85969ebdfa55f178e746be081d1cb4f94011f69eb7c 2024-09-21T10:28:43.201 INFO:teuthology.orchestra.run.smithi123.stderr:/bin/podman: Copying config sha256:e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-09-21T10:28:43.368 INFO:teuthology.orchestra.run.smithi123.stderr:/bin/podman: Writing manifest to image destination 2024-09-21T10:28:43.496 INFO:teuthology.orchestra.run.smithi123.stderr:/bin/podman: e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-09-21T10:28:43.881 INFO:teuthology.orchestra.run.smithi123.stderr:ceph: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-09-21T10:28:44.428 INFO:teuthology.orchestra.run.smithi123.stderr:Ceph version: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-09-21T10:28:44.428 INFO:teuthology.orchestra.run.smithi123.stderr:Extracting ceph user uid/gid from container image... 2024-09-21T10:28:44.844 INFO:teuthology.orchestra.run.smithi123.stderr:stat: 167 167 2024-09-21T10:28:45.332 INFO:teuthology.orchestra.run.smithi123.stderr:Creating initial keys... 2024-09-21T10:28:45.750 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph-authtool: AQDdn+5mSfiBJxAA3+UZkVGyTFJreabRhITL8A== 2024-09-21T10:28:46.629 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph-authtool: AQDen+5mMiBVIBAAu8C/DJg8wQJWo7ZcV41U/w== 2024-09-21T10:28:47.643 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph-authtool: AQDfn+5mylA7IRAAS0KN9RFa24aBadTAvxmkww== 2024-09-21T10:28:48.166 INFO:teuthology.orchestra.run.smithi123.stderr:Creating initial monmap... 2024-09-21T10:28:48.615 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: monmap file /tmp/monmap 2024-09-21T10:28:48.616 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/monmaptool: setting min_mon_release = octopus 2024-09-21T10:28:48.616 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: set fsid to 3ccd15e6-7804-11ef-baf6-efdc52797490 2024-09-21T10:28:48.616 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-09-21T10:28:49.174 INFO:teuthology.orchestra.run.smithi123.stderr:monmaptool for smithi123 [v2:172.21.15.123:3300,v1:172.21.15.123:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-09-21T10:28:49.174 INFO:teuthology.orchestra.run.smithi123.stderr:setting min_mon_release = octopus 2024-09-21T10:28:49.174 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/monmaptool: set fsid to 3ccd15e6-7804-11ef-baf6-efdc52797490 2024-09-21T10:28:49.174 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-09-21T10:28:49.174 INFO:teuthology.orchestra.run.smithi123.stderr: 2024-09-21T10:28:49.174 INFO:teuthology.orchestra.run.smithi123.stderr:Creating mon... 2024-09-21T10:28:50.882 INFO:teuthology.orchestra.run.smithi123.stderr:create mon.smithi123 on 2024-09-21T10:28:51.198 INFO:teuthology.orchestra.run.smithi123.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-09-21T10:28:51.543 INFO:teuthology.orchestra.run.smithi123.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-3ccd15e6-7804-11ef-baf6-efdc52797490.target → /etc/systemd/system/ceph-3ccd15e6-7804-11ef-baf6-efdc52797490.target. 2024-09-21T10:28:51.544 INFO:teuthology.orchestra.run.smithi123.stderr:systemctl: Created symlink /etc/systemd/system/ceph.target.wants/ceph-3ccd15e6-7804-11ef-baf6-efdc52797490.target → /etc/systemd/system/ceph-3ccd15e6-7804-11ef-baf6-efdc52797490.target. 2024-09-21T10:28:52.073 INFO:teuthology.orchestra.run.smithi123.stderr:systemctl: Failed to reset failed state of unit ceph-3ccd15e6-7804-11ef-baf6-efdc52797490@mon.smithi123.service: Unit ceph-3ccd15e6-7804-11ef-baf6-efdc52797490@mon.smithi123.service not loaded. 2024-09-21T10:28:52.081 INFO:teuthology.orchestra.run.smithi123.stderr:systemctl: Created symlink /etc/systemd/system/ceph-3ccd15e6-7804-11ef-baf6-efdc52797490.target.wants/ceph-3ccd15e6-7804-11ef-baf6-efdc52797490@mon.smithi123.service → /etc/systemd/system/ceph-3ccd15e6-7804-11ef-baf6-efdc52797490@.service. 2024-09-21T10:28:52.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:28:52 smithi123 podman[23350]: 2024-09-21 10:28:52.464575534 +0000 UTC m=+0.015863283 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-09-21T10:28:53.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:28:53 smithi123 podman[23350]: 2024-09-21 10:28:53.45911521 +0000 UTC m=+1.010402930 container create 0336a5091b886d1c21e1fed31c07ddf172f5078ec6a3968a96aa62087600d0e7 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-mon-smithi123, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, version=8, io.buildah.version=1.19.8, ceph=True, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, architecture=x86_64, GIT_CLEAN=True, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream) 2024-09-21T10:28:55.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:28:55 smithi123 podman[23350]: 2024-09-21 10:28:55.353470075 +0000 UTC m=+2.904757811 container init 0336a5091b886d1c21e1fed31c07ddf172f5078ec6a3968a96aa62087600d0e7 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-mon-smithi123, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, io.buildah.version=1.19.8, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8, ceph=True, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-09-21T10:28:55.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:28:55 smithi123 podman[23350]: 2024-09-21 10:28:55.357348598 +0000 UTC m=+2.908636319 container start 0336a5091b886d1c21e1fed31c07ddf172f5078ec6a3968a96aa62087600d0e7 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-mon-smithi123, distribution-scope=public, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, ceph=True, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, GIT_CLEAN=True, release=754, build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8) 2024-09-21T10:28:55.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:28:55 smithi123 ceph-mon[23368]: mkfs 3ccd15e6-7804-11ef-baf6-efdc52797490 2024-09-21T10:28:55.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:28:55 smithi123 ceph-mon[23368]: mon.smithi123 is new leader, mons smithi123 in quorum (ranks 0) 2024-09-21T10:28:56.292 INFO:teuthology.orchestra.run.smithi123.stderr:systemctl: disabled 2024-09-21T10:28:56.300 INFO:teuthology.orchestra.run.smithi123.stderr:systemctl: inactive 2024-09-21T10:28:56.301 INFO:teuthology.orchestra.run.smithi123.stderr:firewalld.service is not enabled 2024-09-21T10:28:56.301 INFO:teuthology.orchestra.run.smithi123.stderr:Not possible to enable service . firewalld.service is not available 2024-09-21T10:28:56.301 INFO:teuthology.orchestra.run.smithi123.stderr:Waiting for mon to start... 2024-09-21T10:28:56.301 INFO:teuthology.orchestra.run.smithi123.stderr:Waiting for mon... 2024-09-21T10:28:56.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:28:56 smithi123 bash[23350]: 0336a5091b886d1c21e1fed31c07ddf172f5078ec6a3968a96aa62087600d0e7 2024-09-21T10:28:56.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:28:56 smithi123 systemd[1]: Started Ceph mon.smithi123 for 3ccd15e6-7804-11ef-baf6-efdc52797490. 2024-09-21T10:28:56.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:28:56 smithi123 ceph-mon[23368]: mon.smithi123 is new leader, mons smithi123 in quorum (ranks 0) 2024-09-21T10:28:56.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:28:56 smithi123 ceph-mon[23368]: monmap e1: 1 mons at {smithi123=[v2:172.21.15.123:3300/0,v1:172.21.15.123:6789/0]} 2024-09-21T10:28:56.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:28:56 smithi123 ceph-mon[23368]: fsmap 2024-09-21T10:28:56.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:28:56 smithi123 ceph-mon[23368]: osdmap e1: 0 total, 0 up, 0 in 2024-09-21T10:28:56.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:28:56 smithi123 ceph-mon[23368]: mgrmap e1: no daemons active 2024-09-21T10:28:59.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:28:59 smithi123 ceph-mon[23368]: from='client.? 172.21.15.123:0/31604367' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-21T10:29:00.246 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: cluster: 2024-09-21T10:29:00.246 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: id: 3ccd15e6-7804-11ef-baf6-efdc52797490 2024-09-21T10:29:00.246 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: health: HEALTH_OK 2024-09-21T10:29:00.246 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: 2024-09-21T10:29:00.246 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: services: 2024-09-21T10:29:00.246 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: mon: 1 daemons, quorum smithi123 (age 4s) 2024-09-21T10:29:00.246 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: mgr: no daemons active 2024-09-21T10:29:00.246 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: osd: 0 osds: 0 up, 0 in 2024-09-21T10:29:00.247 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: 2024-09-21T10:29:00.247 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: data: 2024-09-21T10:29:00.247 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: pools: 0 pools, 0 pgs 2024-09-21T10:29:00.247 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: objects: 0 objects, 0 B 2024-09-21T10:29:00.247 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: usage: 0 B used, 0 B / 0 B avail 2024-09-21T10:29:00.247 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: pgs: 2024-09-21T10:29:00.247 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: 2024-09-21T10:29:02.315 INFO:teuthology.orchestra.run.smithi123.stderr:mon is available 2024-09-21T10:29:02.315 INFO:teuthology.orchestra.run.smithi123.stderr:Assimilating anything we can from ceph.conf... 2024-09-21T10:29:02.861 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: 2024-09-21T10:29:02.862 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: [global] 2024-09-21T10:29:02.862 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: fsid = 3ccd15e6-7804-11ef-baf6-efdc52797490 2024-09-21T10:29:02.862 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: mon_host = [v2:172.21.15.123:3300,v1:172.21.15.123:6789] 2024-09-21T10:29:02.862 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-09-21T10:29:02.862 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-09-21T10:29:02.862 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-09-21T10:29:02.862 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-09-21T10:29:02.862 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: 2024-09-21T10:29:02.862 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: [mgr] 2024-09-21T10:29:02.862 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-09-21T10:29:02.862 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: 2024-09-21T10:29:02.862 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: [osd] 2024-09-21T10:29:02.862 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-09-21T10:29:02.863 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-09-21T10:29:03.100 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:02 smithi123 ceph-mon[23368]: from='client.? 172.21.15.123:0/794343524' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-09-21T10:29:03.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:02 smithi123 ceph-mon[23368]: from='client.? 172.21.15.123:0/794343524' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-09-21T10:29:03.370 INFO:teuthology.orchestra.run.smithi123.stderr:Generating new minimal ceph.conf... 2024-09-21T10:29:04.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:03 smithi123 ceph-mon[23368]: from='client.? 172.21.15.123:0/3777712434' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:29:04.427 INFO:teuthology.orchestra.run.smithi123.stderr:Restarting the monitor... 2024-09-21T10:29:04.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:04 smithi123 systemd[1]: Stopping Ceph mon.smithi123 for 3ccd15e6-7804-11ef-baf6-efdc52797490... 2024-09-21T10:29:04.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:04 smithi123 bash[23589]: Error: no container with name or ID "ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-mon.smithi123" found: no such container 2024-09-21T10:29:04.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:04 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-mon-smithi123[23364]: 2024-09-21T10:29:04.558+0000 7fb784523700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi123 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:29:04.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:04 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-mon-smithi123[23364]: 2024-09-21T10:29:04.558+0000 7fb784523700 -1 mon.smithi123@0(leader) e1 *** Got Signal Terminated *** 2024-09-21T10:29:04.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:04 smithi123 podman[23601]: 2024-09-21 10:29:04.579307034 +0000 UTC m=+0.104420619 container died 0336a5091b886d1c21e1fed31c07ddf172f5078ec6a3968a96aa62087600d0e7 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-mon-smithi123, com.redhat.component=centos-stream-container, name=centos-stream, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, architecture=x86_64, distribution-scope=public, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, version=8, build-date=2022-05-03T08:36:31.336870, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD) 2024-09-21T10:29:05.128 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:04 smithi123 podman[23601]: 2024-09-21 10:29:04.87350731 +0000 UTC m=+0.398620889 container cleanup 0336a5091b886d1c21e1fed31c07ddf172f5078ec6a3968a96aa62087600d0e7 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-mon-smithi123, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, version=8, GIT_CLEAN=True, name=centos-stream, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, ceph=True, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, vcs-type=git, RELEASE=HEAD, io.buildah.version=1.19.8, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-09-21T10:29:05.128 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:04 smithi123 bash[23601]: ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-mon-smithi123 2024-09-21T10:29:05.129 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:04 smithi123 bash[23627]: Error: no container with name or ID "ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-mon.smithi123" found: no such container 2024-09-21T10:29:05.129 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 podman[23613]: 2024-09-21 10:29:05.084061217 +0000 UTC m=+0.507780873 container remove 0336a5091b886d1c21e1fed31c07ddf172f5078ec6a3968a96aa62087600d0e7 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-mon-smithi123, ceph=True, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, io.buildah.version=1.19.8, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, name=centos-stream, vcs-type=git, distribution-scope=public, GIT_CLEAN=True) 2024-09-21T10:29:05.129 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 systemd[1]: ceph-3ccd15e6-7804-11ef-baf6-efdc52797490@mon.smithi123.service: Deactivated successfully. 2024-09-21T10:29:05.130 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 systemd[1]: Stopped Ceph mon.smithi123 for 3ccd15e6-7804-11ef-baf6-efdc52797490. 2024-09-21T10:29:05.130 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 systemd[1]: Starting Ceph mon.smithi123 for 3ccd15e6-7804-11ef-baf6-efdc52797490... 2024-09-21T10:29:05.400 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 podman[23690]: 2024-09-21 10:29:05.301647873 +0000 UTC m=+0.107582563 container create 6ad9a8bc53b7409b926333dae2fdd2583283bec41caf28164c54db139d92f6a1 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-mon-smithi123, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, ceph=True, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, release=754, distribution-scope=public, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, version=8, GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=) 2024-09-21T10:29:05.400 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 podman[23690]: 2024-09-21 10:29:05.210504741 +0000 UTC m=+0.016439391 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-09-21T10:29:05.642 INFO:teuthology.orchestra.run.smithi123.stderr:Setting mon public_network to 172.21.0.0/20 2024-09-21T10:29:05.757 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 podman[23690]: 2024-09-21 10:29:05.511954607 +0000 UTC m=+0.317889248 container init 6ad9a8bc53b7409b926333dae2fdd2583283bec41caf28164c54db139d92f6a1 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-mon-smithi123, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, io.openshift.expose-services=, RELEASE=HEAD, distribution-scope=public, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc.) 2024-09-21T10:29:05.757 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 podman[23690]: 2024-09-21 10:29:05.515179492 +0000 UTC m=+0.321114136 container start 6ad9a8bc53b7409b926333dae2fdd2583283bec41caf28164c54db139d92f6a1 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-mon-smithi123, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, ceph=True, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, version=8, distribution-scope=public, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870) 2024-09-21T10:29:05.757 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: set uid:gid to 167:167 (ceph:ceph) 2024-09-21T10:29:05.757 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-09-21T10:29:05.757 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: pidfile_write: ignore empty --pid-file 2024-09-21T10:29:05.757 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: load: jerasure load: lrc 2024-09-21T10:29:05.757 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: RocksDB version: 6.15.5 2024-09-21T10:29:05.758 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-09-21T10:29:05.758 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Compile date Apr 18 2022 2024-09-21T10:29:05.758 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: DB SUMMARY 2024-09-21T10:29:05.758 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: DB Session ID: 8R7SGB0Z09OMHC3CX7F2 2024-09-21T10:29:05.758 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: CURRENT file: CURRENT 2024-09-21T10:29:05.758 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: IDENTITY file: IDENTITY 2024-09-21T10:29:05.758 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-09-21T10:29:05.758 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi123/store.db dir, Total Num: 1, files: 000008.sst 2024-09-21T10:29:05.759 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi123/store.db: 000010.log size: 83523 ; 2024-09-21T10:29:05.759 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.error_if_exists: 0 2024-09-21T10:29:05.759 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.create_if_missing: 0 2024-09-21T10:29:05.759 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.paranoid_checks: 1 2024-09-21T10:29:05.759 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-21T10:29:05.759 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.env: 0x55edc10c6860 2024-09-21T10:29:05.759 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.fs: Posix File System 2024-09-21T10:29:05.759 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.info_log: 0x55edc1d5ffa0 2024-09-21T10:29:05.759 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.max_file_opening_threads: 16 2024-09-21T10:29:05.759 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.statistics: (nil) 2024-09-21T10:29:05.759 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.use_fsync: 0 2024-09-21T10:29:05.759 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.max_log_file_size: 0 2024-09-21T10:29:05.760 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-21T10:29:05.760 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-21T10:29:05.760 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.keep_log_file_num: 1000 2024-09-21T10:29:05.760 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.recycle_log_file_num: 0 2024-09-21T10:29:05.760 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.allow_fallocate: 1 2024-09-21T10:29:05.760 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.allow_mmap_reads: 0 2024-09-21T10:29:05.760 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.allow_mmap_writes: 0 2024-09-21T10:29:05.760 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.use_direct_reads: 0 2024-09-21T10:29:05.760 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-21T10:29:05.760 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.create_missing_column_families: 0 2024-09-21T10:29:05.760 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.db_log_dir: 2024-09-21T10:29:05.760 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-smithi123/store.db 2024-09-21T10:29:05.760 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-21T10:29:05.761 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-21T10:29:05.761 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-21T10:29:05.761 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-21T10:29:05.761 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-21T10:29:05.761 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-21T10:29:05.761 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.advise_random_on_open: 1 2024-09-21T10:29:05.761 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.db_write_buffer_size: 0 2024-09-21T10:29:05.761 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.write_buffer_manager: 0x55edc1e502d0 2024-09-21T10:29:05.761 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-21T10:29:05.761 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-09-21T10:29:05.761 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-21T10:29:05.761 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-21T10:29:05.761 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.rate_limiter: (nil) 2024-09-21T10:29:05.761 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-21T10:29:05.762 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.wal_recovery_mode: 2 2024-09-21T10:29:05.762 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.enable_thread_tracking: 0 2024-09-21T10:29:05.762 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.enable_pipelined_write: 0 2024-09-21T10:29:05.762 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.unordered_write: 0 2024-09-21T10:29:05.762 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-21T10:29:05.762 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-21T10:29:05.762 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-21T10:29:05.762 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-21T10:29:05.762 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.row_cache: None 2024-09-21T10:29:05.762 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.wal_filter: None 2024-09-21T10:29:05.762 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-21T10:29:05.762 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.allow_ingest_behind: 0 2024-09-21T10:29:05.762 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.preserve_deletes: 0 2024-09-21T10:29:05.763 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.two_write_queues: 0 2024-09-21T10:29:05.763 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.manual_wal_flush: 0 2024-09-21T10:29:05.763 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.atomic_flush: 0 2024-09-21T10:29:05.763 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-21T10:29:05.763 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-21T10:29:05.763 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-21T10:29:05.763 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.log_readahead_size: 0 2024-09-21T10:29:05.763 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-21T10:29:05.763 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.best_efforts_recovery: 0 2024-09-21T10:29:05.763 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-21T10:29:05.763 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-21T10:29:05.763 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.allow_data_in_errors: 0 2024-09-21T10:29:05.764 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.db_host_id: __hostname__ 2024-09-21T10:29:05.764 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.max_background_jobs: 2 2024-09-21T10:29:05.764 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.max_background_compactions: -1 2024-09-21T10:29:05.764 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.max_subcompactions: 1 2024-09-21T10:29:05.764 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-21T10:29:05.764 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-21T10:29:05.764 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-21T10:29:05.764 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.max_total_wal_size: 0 2024-09-21T10:29:05.764 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-21T10:29:05.764 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-21T10:29:05.764 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-21T10:29:05.764 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-21T10:29:05.764 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.max_open_files: -1 2024-09-21T10:29:05.764 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.bytes_per_sync: 0 2024-09-21T10:29:05.765 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-21T10:29:05.765 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-21T10:29:05.765 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.compaction_readahead_size: 0 2024-09-21T10:29:05.765 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.max_background_flushes: -1 2024-09-21T10:29:05.765 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Compression algorithms supported: 2024-09-21T10:29:05.765 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-21T10:29:05.765 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: kZSTD supported: 0 2024-09-21T10:29:05.765 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: kXpressCompression supported: 0 2024-09-21T10:29:05.765 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: kLZ4HCCompression supported: 1 2024-09-21T10:29:05.765 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: kLZ4Compression supported: 1 2024-09-21T10:29:05.765 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: kBZip2Compression supported: 0 2024-09-21T10:29:05.765 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: kZlibCompression supported: 1 2024-09-21T10:29:05.766 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: kSnappyCompression supported: 1 2024-09-21T10:29:05.766 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-21T10:29:05.766 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi123/store.db/MANIFEST-000009 2024-09-21T10:29:05.766 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-09-21T10:29:05.766 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-21T10:29:05.766 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.merge_operator: 2024-09-21T10:29:05.766 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.compaction_filter: None 2024-09-21T10:29:05.766 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.compaction_filter_factory: None 2024-09-21T10:29:05.766 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.sst_partitioner_factory: None 2024-09-21T10:29:05.766 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-21T10:29:05.766 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-21T10:29:05.766 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55edc1d2bd00) 2024-09-21T10:29:05.767 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: cache_index_and_filter_blocks: 1 2024-09-21T10:29:05.767 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-21T10:29:05.767 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-21T10:29:05.767 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: pin_top_level_index_and_filter: 1 2024-09-21T10:29:05.767 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: index_type: 0 2024-09-21T10:29:05.767 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: data_block_index_type: 0 2024-09-21T10:29:05.767 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: index_shortening: 1 2024-09-21T10:29:05.767 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-21T10:29:05.767 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: hash_index_allow_collision: 1 2024-09-21T10:29:05.767 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: checksum: 1 2024-09-21T10:29:05.767 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: no_block_cache: 0 2024-09-21T10:29:05.767 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: block_cache: 0x55edc1d96170 2024-09-21T10:29:05.768 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: block_cache_name: BinnedLRUCache 2024-09-21T10:29:05.768 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: block_cache_options: 2024-09-21T10:29:05.768 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: capacity : 536870912 2024-09-21T10:29:05.769 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: num_shard_bits : 4 2024-09-21T10:29:05.769 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: strict_capacity_limit : 0 2024-09-21T10:29:05.769 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: high_pri_pool_ratio: 0.000 2024-09-21T10:29:05.769 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: block_cache_compressed: (nil) 2024-09-21T10:29:05.769 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: persistent_cache: (nil) 2024-09-21T10:29:05.769 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: block_size: 4096 2024-09-21T10:29:05.769 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: block_size_deviation: 10 2024-09-21T10:29:05.769 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: block_restart_interval: 16 2024-09-21T10:29:05.769 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: index_block_restart_interval: 1 2024-09-21T10:29:05.769 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: metadata_block_size: 4096 2024-09-21T10:29:05.770 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: partition_filters: 0 2024-09-21T10:29:05.770 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: use_delta_encoding: 1 2024-09-21T10:29:05.770 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-09-21T10:29:05.770 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: whole_key_filtering: 1 2024-09-21T10:29:05.770 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: verify_compression: 0 2024-09-21T10:29:05.770 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: read_amp_bytes_per_bit: 0 2024-09-21T10:29:05.770 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: format_version: 4 2024-09-21T10:29:05.770 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: enable_index_compression: 1 2024-09-21T10:29:05.770 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: block_align: 0 2024-09-21T10:29:05.770 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.write_buffer_size: 33554432 2024-09-21T10:29:05.770 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.max_write_buffer_number: 2 2024-09-21T10:29:05.770 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.compression: NoCompression 2024-09-21T10:29:05.770 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.bottommost_compression: Disabled 2024-09-21T10:29:05.771 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.prefix_extractor: nullptr 2024-09-21T10:29:05.771 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-21T10:29:05.771 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.num_levels: 7 2024-09-21T10:29:05.771 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-21T10:29:05.771 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-21T10:29:05.771 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-21T10:29:05.771 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-21T10:29:05.771 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-21T10:29:05.771 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-21T10:29:05.771 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-21T10:29:05.771 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-21T10:29:05.771 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-21T10:29:05.771 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-21T10:29:05.771 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-21T10:29:05.772 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.compression_opts.level: 32767 2024-09-21T10:29:05.772 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.compression_opts.strategy: 0 2024-09-21T10:29:05.772 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-21T10:29:05.772 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-21T10:29:05.772 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-21T10:29:05.772 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.compression_opts.enabled: false 2024-09-21T10:29:05.772 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-21T10:29:05.772 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-21T10:29:05.772 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-21T10:29:05.772 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.target_file_size_base: 67108864 2024-09-21T10:29:05.772 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-21T10:29:05.772 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-21T10:29:05.772 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-21T10:29:05.772 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-21T10:29:05.772 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-21T10:29:05.773 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-21T10:29:05.773 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-21T10:29:05.773 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-21T10:29:05.773 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-21T10:29:05.773 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-21T10:29:05.773 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-21T10:29:05.773 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-21T10:29:05.773 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-21T10:29:05.773 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.arena_block_size: 4194304 2024-09-21T10:29:05.773 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-21T10:29:05.773 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-21T10:29:05.773 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-09-21T10:29:05.773 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.disable_auto_compactions: 0 2024-09-21T10:29:05.773 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-21T10:29:05.773 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-21T10:29:05.774 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-21T10:29:05.774 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-21T10:29:05.774 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-21T10:29:05.774 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-21T10:29:05.774 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-21T10:29:05.774 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-21T10:29:05.774 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-21T10:29:05.774 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-21T10:29:05.774 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.table_properties_collectors: 2024-09-21T10:29:05.774 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.inplace_update_support: 0 2024-09-21T10:29:05.774 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-21T10:29:05.774 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-21T10:29:05.774 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-21T10:29:05.775 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-21T10:29:05.775 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.bloom_locality: 0 2024-09-21T10:29:05.775 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.max_successive_merges: 0 2024-09-21T10:29:05.775 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-21T10:29:05.775 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.paranoid_file_checks: 0 2024-09-21T10:29:05.775 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.force_consistency_checks: 1 2024-09-21T10:29:05.775 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.report_bg_io_stats: 0 2024-09-21T10:29:05.775 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.ttl: 2592000 2024-09-21T10:29:05.775 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-21T10:29:05.775 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.enable_blob_files: false 2024-09-21T10:29:05.775 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.min_blob_size: 0 2024-09-21T10:29:05.775 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.blob_file_size: 268435456 2024-09-21T10:29:05.775 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-21T10:29:05.775 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-21T10:29:05.776 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-21T10:29:05.776 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi123/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-21T10:29:05.776 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 5 2024-09-21T10:29:05.776 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: [db/version_set.cc:4083] Creating manifest 13 2024-09-21T10:29:05.776 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726914545540967, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-09-21T10:29:05.777 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #10 mode 2 2024-09-21T10:29:05.777 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-09-21T10:29:05.777 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726914545541662, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 79851, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 78096, "index_size": 243, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 9930, "raw_average_key_size": 46, "raw_value_size": 72347, "raw_average_value_size": 341, "num_data_blocks": 11, "num_entries": 212, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1726914545, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "54181fdc-f81b-46d5-93de-b9b3d753f2c8", "db_session_id": "8R7SGB0Z09OMHC3CX7F2"}} 2024-09-21T10:29:05.778 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: [db/version_set.cc:4083] Creating manifest 15 2024-09-21T10:29:05.778 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726914545541992, "job": 1, "event": "recovery_finished"} 2024-09-21T10:29:05.778 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-smithi123/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-21T10:29:05.778 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x55edc1d7ca80 2024-09-21T10:29:05.778 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: DB pointer 0x55edc1d8c000 2024-09-21T10:29:05.778 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-09-21T10:29:05.778 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-09-21T10:29:05.778 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: ** DB Stats ** 2024-09-21T10:29:05.778 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T10:29:05.778 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-21T10:29:05.778 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-21T10:29:05.778 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T10:29:05.778 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-21T10:29:05.778 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-09-21T10:29:05.778 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T10:29:05.779 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: 2024-09-21T10:29:05.779 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: ** Compaction Stats [default] ** 2024-09-21T10:29:05.779 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-21T10:29:05.779 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-21T10:29:05.779 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: L0 2/0 79.78 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 165.2 0.00 0.00 1 0.000 0 0 2024-09-21T10:29:05.779 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Sum 2/0 79.78 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 165.2 0.00 0.00 1 0.000 0 0 2024-09-21T10:29:05.779 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 165.2 0.00 0.00 1 0.000 0 0 2024-09-21T10:29:05.779 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: 2024-09-21T10:29:05.779 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: ** Compaction Stats [default] ** 2024-09-21T10:29:05.779 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-21T10:29:05.779 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-21T10:29:05.779 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 165.2 0.00 0.00 1 0.000 0 0 2024-09-21T10:29:05.779 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T10:29:05.780 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-21T10:29:05.780 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-21T10:29:05.780 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-21T10:29:05.780 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-21T10:29:05.780 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-21T10:29:05.780 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Cumulative compaction: 0.00 GB write, 25.21 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T10:29:05.780 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Interval compaction: 0.00 GB write, 25.21 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T10:29:05.780 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-21T10:29:05.780 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: 2024-09-21T10:29:05.780 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-21T10:29:05.780 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: 2024-09-21T10:29:05.780 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: ** Compaction Stats [default] ** 2024-09-21T10:29:05.780 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-21T10:29:05.780 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-21T10:29:05.781 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: L0 2/0 79.78 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 165.2 0.00 0.00 1 0.000 0 0 2024-09-21T10:29:05.781 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Sum 2/0 79.78 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 165.2 0.00 0.00 1 0.000 0 0 2024-09-21T10:29:05.781 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-21T10:29:05.781 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: 2024-09-21T10:29:05.781 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: ** Compaction Stats [default] ** 2024-09-21T10:29:05.781 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-21T10:29:05.781 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-21T10:29:05.781 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 165.2 0.00 0.00 1 0.000 0 0 2024-09-21T10:29:05.781 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T10:29:05.781 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-21T10:29:05.781 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-21T10:29:05.782 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-21T10:29:05.782 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-21T10:29:05.782 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-21T10:29:05.782 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Cumulative compaction: 0.00 GB write, 24.94 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T10:29:05.782 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T10:29:05.782 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-21T10:29:05.782 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: 2024-09-21T10:29:05.782 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-21T10:29:05.782 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: starting mon.smithi123 rank 0 at public addrs [v2:172.21.15.123:3300/0,v1:172.21.15.123:6789/0] at bind addrs [v2:172.21.15.123:3300/0,v1:172.21.15.123:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi123 fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 2024-09-21T10:29:05.782 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: mon.smithi123@-1(???) e1 preinit fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 2024-09-21T10:29:05.782 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: mon.smithi123@-1(???).mds e1 new map 2024-09-21T10:29:05.782 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: mon.smithi123@-1(???).mds e1 print_map 2024-09-21T10:29:05.782 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: e1 2024-09-21T10:29:05.783 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-21T10:29:05.783 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-21T10:29:05.783 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: legacy client fscid: -1 2024-09-21T10:29:05.783 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: 2024-09-21T10:29:05.783 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: No filesystems configured 2024-09-21T10:29:05.783 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: mon.smithi123@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-09-21T10:29:05.783 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: mon.smithi123@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-21T10:29:05.783 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: mon.smithi123@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-21T10:29:05.783 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: mon.smithi123@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-21T10:29:05.783 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: mon.smithi123@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-09-21T10:29:05.783 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: mon.smithi123 is new leader, mons smithi123 in quorum (ranks 0) 2024-09-21T10:29:05.783 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: monmap e1: 1 mons at {smithi123=[v2:172.21.15.123:3300/0,v1:172.21.15.123:6789/0]} 2024-09-21T10:29:05.784 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: fsmap 2024-09-21T10:29:05.784 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: osdmap e1: 0 total, 0 up, 0 in 2024-09-21T10:29:05.784 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 ceph-mon[23709]: mgrmap e1: no daemons active 2024-09-21T10:29:05.784 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 bash[23690]: 6ad9a8bc53b7409b926333dae2fdd2583283bec41caf28164c54db139d92f6a1 2024-09-21T10:29:05.784 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:05 smithi123 systemd[1]: Started Ceph mon.smithi123 for 3ccd15e6-7804-11ef-baf6-efdc52797490. 2024-09-21T10:29:06.673 INFO:teuthology.orchestra.run.smithi123.stderr:Wrote config to /etc/ceph/ceph.conf 2024-09-21T10:29:06.674 INFO:teuthology.orchestra.run.smithi123.stderr:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-09-21T10:29:06.674 INFO:teuthology.orchestra.run.smithi123.stderr:Creating mgr... 2024-09-21T10:29:06.674 INFO:teuthology.orchestra.run.smithi123.stderr:Verifying port 9283 ... 2024-09-21T10:29:06.920 INFO:teuthology.orchestra.run.smithi123.stderr:systemctl: Failed to reset failed state of unit ceph-3ccd15e6-7804-11ef-baf6-efdc52797490@mgr.smithi123.stnuit.service: Unit ceph-3ccd15e6-7804-11ef-baf6-efdc52797490@mgr.smithi123.stnuit.service not loaded. 2024-09-21T10:29:06.928 INFO:teuthology.orchestra.run.smithi123.stderr:systemctl: Created symlink /etc/systemd/system/ceph-3ccd15e6-7804-11ef-baf6-efdc52797490.target.wants/ceph-3ccd15e6-7804-11ef-baf6-efdc52797490@mgr.smithi123.stnuit.service → /etc/systemd/system/ceph-3ccd15e6-7804-11ef-baf6-efdc52797490@.service. 2024-09-21T10:29:07.294 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:07 smithi123 ceph-mon[23709]: from='client.? 172.21.15.123:0/4055434589' entity='client.admin' 2024-09-21T10:29:08.284 INFO:teuthology.orchestra.run.smithi123.stderr:systemctl: disabled 2024-09-21T10:29:08.291 INFO:teuthology.orchestra.run.smithi123.stderr:systemctl: inactive 2024-09-21T10:29:08.292 INFO:teuthology.orchestra.run.smithi123.stderr:firewalld.service is not enabled 2024-09-21T10:29:08.292 INFO:teuthology.orchestra.run.smithi123.stderr:Not possible to enable service . firewalld.service is not available 2024-09-21T10:29:08.300 INFO:teuthology.orchestra.run.smithi123.stderr:systemctl: disabled 2024-09-21T10:29:08.307 INFO:teuthology.orchestra.run.smithi123.stderr:systemctl: inactive 2024-09-21T10:29:08.308 INFO:teuthology.orchestra.run.smithi123.stderr:firewalld.service is not enabled 2024-09-21T10:29:08.308 INFO:teuthology.orchestra.run.smithi123.stderr:Not possible to open ports <[9283]>. firewalld.service is not available 2024-09-21T10:29:08.308 INFO:teuthology.orchestra.run.smithi123.stderr:Waiting for mgr to start... 2024-09-21T10:29:08.308 INFO:teuthology.orchestra.run.smithi123.stderr:Waiting for mgr... 2024-09-21T10:29:08.821 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: 2024-09-21T10:29:08.821 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: { 2024-09-21T10:29:08.821 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "fsid": "3ccd15e6-7804-11ef-baf6-efdc52797490", 2024-09-21T10:29:08.821 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "health": { 2024-09-21T10:29:08.822 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-21T10:29:08.822 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "checks": {}, 2024-09-21T10:29:08.822 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "mutes": [] 2024-09-21T10:29:08.822 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: }, 2024-09-21T10:29:08.822 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-21T10:29:08.822 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "quorum": [ 2024-09-21T10:29:08.822 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: 0 2024-09-21T10:29:08.822 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: ], 2024-09-21T10:29:08.822 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-21T10:29:08.822 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "smithi123" 2024-09-21T10:29:08.822 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: ], 2024-09-21T10:29:08.822 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "quorum_age": 3, 2024-09-21T10:29:08.822 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "monmap": { 2024-09-21T10:29:08.822 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-21T10:29:08.823 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-21T10:29:08.823 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-21T10:29:08.823 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: }, 2024-09-21T10:29:08.823 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "osdmap": { 2024-09-21T10:29:08.823 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-21T10:29:08.823 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-21T10:29:08.823 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-21T10:29:08.823 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-21T10:29:08.823 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-21T10:29:08.826 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-21T10:29:08.826 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-21T10:29:08.826 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: }, 2024-09-21T10:29:08.826 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "pgmap": { 2024-09-21T10:29:08.826 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-21T10:29:08.826 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-21T10:29:08.826 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-21T10:29:08.826 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-21T10:29:08.826 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-21T10:29:08.827 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-21T10:29:08.827 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-21T10:29:08.827 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-21T10:29:08.827 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: }, 2024-09-21T10:29:08.827 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "fsmap": { 2024-09-21T10:29:08.827 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-21T10:29:08.827 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-21T10:29:08.827 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-21T10:29:08.827 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: }, 2024-09-21T10:29:08.827 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-21T10:29:08.827 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "available": false, 2024-09-21T10:29:08.827 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-21T10:29:08.827 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "modules": [ 2024-09-21T10:29:08.827 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "iostat", 2024-09-21T10:29:08.828 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "nfs", 2024-09-21T10:29:08.828 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "restful" 2024-09-21T10:29:08.828 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: ], 2024-09-21T10:29:08.828 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "services": {} 2024-09-21T10:29:08.828 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: }, 2024-09-21T10:29:08.828 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "servicemap": { 2024-09-21T10:29:08.828 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-21T10:29:08.828 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "modified": "2024-09-21T10:28:55.400410+0000", 2024-09-21T10:29:08.828 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "services": {} 2024-09-21T10:29:08.828 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: }, 2024-09-21T10:29:08.828 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-21T10:29:08.828 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: } 2024-09-21T10:29:09.100 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:08 smithi123 ceph-mon[23709]: from='client.? 172.21.15.123:0/539857450' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-21T10:29:09.396 INFO:teuthology.orchestra.run.smithi123.stderr:mgr not available, waiting (1/15)... 2024-09-21T10:29:11.931 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: 2024-09-21T10:29:11.931 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: { 2024-09-21T10:29:11.931 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "fsid": "3ccd15e6-7804-11ef-baf6-efdc52797490", 2024-09-21T10:29:11.931 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "health": { 2024-09-21T10:29:11.932 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-21T10:29:11.932 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "checks": {}, 2024-09-21T10:29:11.932 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "mutes": [] 2024-09-21T10:29:11.932 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: }, 2024-09-21T10:29:11.932 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-21T10:29:11.932 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "quorum": [ 2024-09-21T10:29:11.932 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: 0 2024-09-21T10:29:11.932 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: ], 2024-09-21T10:29:11.932 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-21T10:29:11.932 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "smithi123" 2024-09-21T10:29:11.932 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: ], 2024-09-21T10:29:11.932 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "quorum_age": 6, 2024-09-21T10:29:11.932 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "monmap": { 2024-09-21T10:29:11.932 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-21T10:29:11.932 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-21T10:29:11.933 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-21T10:29:11.933 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: }, 2024-09-21T10:29:11.933 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "osdmap": { 2024-09-21T10:29:11.933 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-21T10:29:11.933 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-21T10:29:11.933 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-21T10:29:11.933 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-21T10:29:11.933 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-21T10:29:11.936 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-21T10:29:11.936 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-21T10:29:11.936 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: }, 2024-09-21T10:29:11.936 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "pgmap": { 2024-09-21T10:29:11.936 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-21T10:29:11.936 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-21T10:29:11.936 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-21T10:29:11.936 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-21T10:29:11.936 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-21T10:29:11.937 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-21T10:29:11.937 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-21T10:29:11.937 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-21T10:29:11.937 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: }, 2024-09-21T10:29:11.937 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "fsmap": { 2024-09-21T10:29:11.937 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-21T10:29:11.937 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-21T10:29:11.937 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-21T10:29:11.937 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: }, 2024-09-21T10:29:11.937 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-21T10:29:11.937 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "available": false, 2024-09-21T10:29:11.937 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-21T10:29:11.937 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "modules": [ 2024-09-21T10:29:11.938 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "iostat", 2024-09-21T10:29:11.938 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "nfs", 2024-09-21T10:29:11.938 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "restful" 2024-09-21T10:29:11.938 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: ], 2024-09-21T10:29:11.938 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "services": {} 2024-09-21T10:29:11.938 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: }, 2024-09-21T10:29:11.938 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "servicemap": { 2024-09-21T10:29:11.938 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-21T10:29:11.938 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "modified": "2024-09-21T10:28:55.400410+0000", 2024-09-21T10:29:11.938 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "services": {} 2024-09-21T10:29:11.938 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: }, 2024-09-21T10:29:11.938 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-21T10:29:11.938 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: } 2024-09-21T10:29:12.100 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:11 smithi123 ceph-mon[23709]: from='client.? 172.21.15.123:0/4128899848' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-21T10:29:12.511 INFO:teuthology.orchestra.run.smithi123.stderr:mgr not available, waiting (2/15)... 2024-09-21T10:29:14.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:14 smithi123 ceph-mon[23709]: Activating manager daemon smithi123.stnuit 2024-09-21T10:29:14.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:14 smithi123 ceph-mon[23709]: mgrmap e2: smithi123.stnuit(active, starting, since 0.00259054s) 2024-09-21T10:29:14.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:14 smithi123 ceph-mon[23709]: from='mgr.14100 172.21.15.123:0/1421805467' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T10:29:14.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:14 smithi123 ceph-mon[23709]: from='mgr.14100 172.21.15.123:0/1421805467' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T10:29:14.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:14 smithi123 ceph-mon[23709]: from='mgr.14100 172.21.15.123:0/1421805467' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T10:29:14.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:14 smithi123 ceph-mon[23709]: from='mgr.14100 172.21.15.123:0/1421805467' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mon metadata", "id": "smithi123"}]: dispatch 2024-09-21T10:29:14.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:14 smithi123 ceph-mon[23709]: from='mgr.14100 172.21.15.123:0/1421805467' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mgr metadata", "who": "smithi123.stnuit", "id": "smithi123.stnuit"}]: dispatch 2024-09-21T10:29:14.602 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:14 smithi123 ceph-mon[23709]: Manager daemon smithi123.stnuit is now available 2024-09-21T10:29:14.602 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:14 smithi123 ceph-mon[23709]: from='mgr.14100 172.21.15.123:0/1421805467' entity='mgr.smithi123.stnuit' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.stnuit/mirror_snapshot_schedule"}]: dispatch 2024-09-21T10:29:14.602 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:14 smithi123 ceph-mon[23709]: from='mgr.14100 172.21.15.123:0/1421805467' entity='mgr.smithi123.stnuit' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.stnuit/trash_purge_schedule"}]: dispatch 2024-09-21T10:29:14.602 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:14 smithi123 ceph-mon[23709]: from='mgr.14100 172.21.15.123:0/1421805467' entity='mgr.smithi123.stnuit' 2024-09-21T10:29:14.602 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:14 smithi123 ceph-mon[23709]: from='mgr.14100 172.21.15.123:0/1421805467' entity='mgr.smithi123.stnuit' 2024-09-21T10:29:14.602 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:14 smithi123 ceph-mon[23709]: from='mgr.14100 172.21.15.123:0/1421805467' entity='mgr.smithi123.stnuit' 2024-09-21T10:29:15.007 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: 2024-09-21T10:29:15.007 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: { 2024-09-21T10:29:15.007 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "fsid": "3ccd15e6-7804-11ef-baf6-efdc52797490", 2024-09-21T10:29:15.007 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "health": { 2024-09-21T10:29:15.007 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-21T10:29:15.007 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "checks": {}, 2024-09-21T10:29:15.007 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "mutes": [] 2024-09-21T10:29:15.007 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: }, 2024-09-21T10:29:15.008 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-21T10:29:15.008 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "quorum": [ 2024-09-21T10:29:15.008 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: 0 2024-09-21T10:29:15.008 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: ], 2024-09-21T10:29:15.008 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-21T10:29:15.008 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "smithi123" 2024-09-21T10:29:15.008 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: ], 2024-09-21T10:29:15.008 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "quorum_age": 9, 2024-09-21T10:29:15.008 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "monmap": { 2024-09-21T10:29:15.008 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-21T10:29:15.008 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-21T10:29:15.008 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-21T10:29:15.008 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: }, 2024-09-21T10:29:15.008 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "osdmap": { 2024-09-21T10:29:15.008 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-21T10:29:15.009 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-21T10:29:15.009 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-21T10:29:15.009 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-21T10:29:15.009 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-21T10:29:15.012 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-21T10:29:15.012 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-21T10:29:15.012 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: }, 2024-09-21T10:29:15.012 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "pgmap": { 2024-09-21T10:29:15.012 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-21T10:29:15.012 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-21T10:29:15.012 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-21T10:29:15.012 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-21T10:29:15.013 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-21T10:29:15.013 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-21T10:29:15.013 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-21T10:29:15.013 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-21T10:29:15.013 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: }, 2024-09-21T10:29:15.013 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "fsmap": { 2024-09-21T10:29:15.013 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-21T10:29:15.013 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-21T10:29:15.013 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-21T10:29:15.013 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: }, 2024-09-21T10:29:15.013 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-21T10:29:15.013 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "available": false, 2024-09-21T10:29:15.013 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-21T10:29:15.013 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "modules": [ 2024-09-21T10:29:15.014 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "iostat", 2024-09-21T10:29:15.014 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "nfs", 2024-09-21T10:29:15.014 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "restful" 2024-09-21T10:29:15.014 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: ], 2024-09-21T10:29:15.014 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "services": {} 2024-09-21T10:29:15.014 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: }, 2024-09-21T10:29:15.014 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "servicemap": { 2024-09-21T10:29:15.014 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-21T10:29:15.014 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "modified": "2024-09-21T10:28:55.400410+0000", 2024-09-21T10:29:15.014 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "services": {} 2024-09-21T10:29:15.014 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: }, 2024-09-21T10:29:15.014 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-21T10:29:15.014 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: } 2024-09-21T10:29:15.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:15 smithi123 ceph-mon[23709]: from='client.? 172.21.15.123:0/2230485861' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-21T10:29:15.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:15 smithi123 ceph-mon[23709]: mgrmap e3: smithi123.stnuit(active, since 1.00607s) 2024-09-21T10:29:15.641 INFO:teuthology.orchestra.run.smithi123.stderr:mgr not available, waiting (3/15)... 2024-09-21T10:29:17.850 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:17 smithi123 ceph-mon[23709]: mgrmap e4: smithi123.stnuit(active, since 2s) 2024-09-21T10:29:18.302 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: 2024-09-21T10:29:18.302 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: { 2024-09-21T10:29:18.302 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "fsid": "3ccd15e6-7804-11ef-baf6-efdc52797490", 2024-09-21T10:29:18.302 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "health": { 2024-09-21T10:29:18.302 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-21T10:29:18.302 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "checks": {}, 2024-09-21T10:29:18.302 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "mutes": [] 2024-09-21T10:29:18.302 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: }, 2024-09-21T10:29:18.302 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-21T10:29:18.302 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "quorum": [ 2024-09-21T10:29:18.303 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: 0 2024-09-21T10:29:18.303 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: ], 2024-09-21T10:29:18.303 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-21T10:29:18.303 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "smithi123" 2024-09-21T10:29:18.303 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: ], 2024-09-21T10:29:18.303 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "quorum_age": 12, 2024-09-21T10:29:18.303 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "monmap": { 2024-09-21T10:29:18.303 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-21T10:29:18.303 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-21T10:29:18.303 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-21T10:29:18.303 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: }, 2024-09-21T10:29:18.303 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "osdmap": { 2024-09-21T10:29:18.303 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-21T10:29:18.303 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-21T10:29:18.304 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-21T10:29:18.304 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-21T10:29:18.304 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-21T10:29:18.307 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-21T10:29:18.307 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-21T10:29:18.307 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: }, 2024-09-21T10:29:18.307 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "pgmap": { 2024-09-21T10:29:18.307 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-21T10:29:18.307 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-21T10:29:18.307 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-21T10:29:18.307 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-21T10:29:18.307 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-21T10:29:18.308 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-21T10:29:18.308 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-21T10:29:18.308 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-21T10:29:18.308 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: }, 2024-09-21T10:29:18.308 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "fsmap": { 2024-09-21T10:29:18.308 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-21T10:29:18.308 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-21T10:29:18.308 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-21T10:29:18.308 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: }, 2024-09-21T10:29:18.308 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-21T10:29:18.309 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "available": true, 2024-09-21T10:29:18.309 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-21T10:29:18.309 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "modules": [ 2024-09-21T10:29:18.309 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "iostat", 2024-09-21T10:29:18.309 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "nfs", 2024-09-21T10:29:18.309 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "restful" 2024-09-21T10:29:18.309 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: ], 2024-09-21T10:29:18.309 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "services": {} 2024-09-21T10:29:18.309 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: }, 2024-09-21T10:29:18.309 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "servicemap": { 2024-09-21T10:29:18.309 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-21T10:29:18.309 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "modified": "2024-09-21T10:28:55.400410+0000", 2024-09-21T10:29:18.310 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "services": {} 2024-09-21T10:29:18.310 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: }, 2024-09-21T10:29:18.310 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-21T10:29:18.310 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: } 2024-09-21T10:29:18.850 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:18 smithi123 ceph-mon[23709]: from='client.? 172.21.15.123:0/3658483970' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-21T10:29:18.955 INFO:teuthology.orchestra.run.smithi123.stderr:mgr is available 2024-09-21T10:29:19.561 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: 2024-09-21T10:29:19.562 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: [global] 2024-09-21T10:29:19.562 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: fsid = 3ccd15e6-7804-11ef-baf6-efdc52797490 2024-09-21T10:29:19.562 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-09-21T10:29:19.562 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-09-21T10:29:19.562 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-09-21T10:29:19.562 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-09-21T10:29:19.562 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: 2024-09-21T10:29:19.562 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: [mgr] 2024-09-21T10:29:19.562 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-09-21T10:29:19.562 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: 2024-09-21T10:29:19.562 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: [osd] 2024-09-21T10:29:19.562 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-09-21T10:29:19.563 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-09-21T10:29:19.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:19 smithi123 ceph-mon[23709]: from='client.? 172.21.15.123:0/1472292299' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-09-21T10:29:20.136 INFO:teuthology.orchestra.run.smithi123.stderr:Enabling cephadm module... 2024-09-21T10:29:21.850 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:21 smithi123 ceph-mon[23709]: from='client.? 172.21.15.123:0/344045209' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-09-21T10:29:22.740 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:22 smithi123 ceph-mon[23709]: from='client.? 172.21.15.123:0/344045209' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-09-21T10:29:22.740 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:22 smithi123 ceph-mon[23709]: mgrmap e5: smithi123.stnuit(active, since 7s) 2024-09-21T10:29:22.886 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: { 2024-09-21T10:29:22.887 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "epoch": 5, 2024-09-21T10:29:22.887 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "available": true, 2024-09-21T10:29:22.887 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "active_name": "smithi123.stnuit", 2024-09-21T10:29:22.887 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "num_standby": 0 2024-09-21T10:29:22.887 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: } 2024-09-21T10:29:23.598 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:23 smithi123 ceph-mon[23709]: from='client.? 172.21.15.123:0/300531500' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-21T10:29:23.598 INFO:teuthology.orchestra.run.smithi123.stderr:Waiting for the mgr to restart... 2024-09-21T10:29:23.598 INFO:teuthology.orchestra.run.smithi123.stderr:Waiting for mgr epoch 5... 2024-09-21T10:29:28.100 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:27 smithi123 ceph-mon[23709]: Active manager daemon smithi123.stnuit restarted 2024-09-21T10:29:28.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:27 smithi123 ceph-mon[23709]: Activating manager daemon smithi123.stnuit 2024-09-21T10:29:28.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:27 smithi123 ceph-mon[23709]: osdmap e2: 0 total, 0 up, 0 in 2024-09-21T10:29:28.701 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: { 2024-09-21T10:29:28.702 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "mgrmap_epoch": 7, 2024-09-21T10:29:28.702 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "initialized": true 2024-09-21T10:29:28.702 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: } 2024-09-21T10:29:28.971 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:28 smithi123 ceph-mon[23709]: mgrmap e6: smithi123.stnuit(active, starting, since 0.0525601s) 2024-09-21T10:29:28.971 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:28 smithi123 ceph-mon[23709]: from='mgr.14120 172.21.15.123:0/759259812' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mon metadata", "id": "smithi123"}]: dispatch 2024-09-21T10:29:28.971 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:28 smithi123 ceph-mon[23709]: from='mgr.14120 172.21.15.123:0/759259812' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mgr metadata", "who": "smithi123.stnuit", "id": "smithi123.stnuit"}]: dispatch 2024-09-21T10:29:28.971 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:28 smithi123 ceph-mon[23709]: from='mgr.14120 172.21.15.123:0/759259812' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T10:29:28.971 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:28 smithi123 ceph-mon[23709]: from='mgr.14120 172.21.15.123:0/759259812' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T10:29:28.971 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:28 smithi123 ceph-mon[23709]: from='mgr.14120 172.21.15.123:0/759259812' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T10:29:28.971 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:28 smithi123 ceph-mon[23709]: Manager daemon smithi123.stnuit is now available 2024-09-21T10:29:28.971 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:28 smithi123 ceph-mon[23709]: from='mgr.14120 172.21.15.123:0/759259812' entity='mgr.smithi123.stnuit' 2024-09-21T10:29:28.971 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:28 smithi123 ceph-mon[23709]: from='mgr.14120 172.21.15.123:0/759259812' entity='mgr.smithi123.stnuit' 2024-09-21T10:29:28.971 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:28 smithi123 ceph-mon[23709]: from='mgr.14120 172.21.15.123:0/759259812' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:29:28.972 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:28 smithi123 ceph-mon[23709]: from='mgr.14120 172.21.15.123:0/759259812' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:29:28.972 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:28 smithi123 ceph-mon[23709]: from='mgr.14120 172.21.15.123:0/759259812' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:29:28.972 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:28 smithi123 ceph-mon[23709]: from='mgr.14120 172.21.15.123:0/759259812' entity='mgr.smithi123.stnuit' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.stnuit/mirror_snapshot_schedule"}]: dispatch 2024-09-21T10:29:28.972 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:28 smithi123 ceph-mon[23709]: from='mgr.14120 172.21.15.123:0/759259812' entity='mgr.smithi123.stnuit' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.stnuit/trash_purge_schedule"}]: dispatch 2024-09-21T10:29:28.972 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:28 smithi123 ceph-mon[23709]: [21/Sep/2024:10:29:28] ENGINE Bus STARTING 2024-09-21T10:29:28.972 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:28 smithi123 ceph-mon[23709]: [21/Sep/2024:10:29:28] ENGINE Serving on https://172.21.15.123:7150 2024-09-21T10:29:28.972 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:28 smithi123 ceph-mon[23709]: [21/Sep/2024:10:29:28] ENGINE Bus STARTED 2024-09-21T10:29:28.972 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:28 smithi123 ceph-mon[23709]: from='mgr.14120 172.21.15.123:0/759259812' entity='mgr.smithi123.stnuit' 2024-09-21T10:29:28.972 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:28 smithi123 ceph-mon[23709]: from='mgr.14120 172.21.15.123:0/759259812' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:29:29.312 INFO:teuthology.orchestra.run.smithi123.stderr:mgr epoch 5 is available 2024-09-21T10:29:29.312 INFO:teuthology.orchestra.run.smithi123.stderr:Setting orchestrator backend to cephadm... 2024-09-21T10:29:29.949 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:29 smithi123 ceph-mon[23709]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-09-21T10:29:29.949 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:29 smithi123 ceph-mon[23709]: mgrmap e7: smithi123.stnuit(active, since 1.05946s) 2024-09-21T10:29:29.949 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:29 smithi123 ceph-mon[23709]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-09-21T10:29:30.773 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:30 smithi123 ceph-mon[23709]: mgrmap e8: smithi123.stnuit(active, since 2s) 2024-09-21T10:29:30.773 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:30 smithi123 ceph-mon[23709]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:29:30.773 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:30 smithi123 ceph-mon[23709]: from='mgr.14120 172.21.15.123:0/759259812' entity='mgr.smithi123.stnuit' 2024-09-21T10:29:30.773 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:30 smithi123 ceph-mon[23709]: from='mgr.14120 172.21.15.123:0/759259812' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:29:31.158 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: value unchanged 2024-09-21T10:29:31.755 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:31 smithi123 ceph-mon[23709]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:29:31.757 INFO:teuthology.orchestra.run.smithi123.stderr:Generating ssh key... 2024-09-21T10:29:33.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:33 smithi123 ceph-mon[23709]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:29:33.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:33 smithi123 ceph-mon[23709]: Generating ssh key... 2024-09-21T10:29:33.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:33 smithi123 ceph-mon[23709]: from='mgr.14120 172.21.15.123:0/759259812' entity='mgr.smithi123.stnuit' 2024-09-21T10:29:33.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:33 smithi123 ceph-mon[23709]: from='mgr.14120 172.21.15.123:0/759259812' entity='mgr.smithi123.stnuit' 2024-09-21T10:29:33.609 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDZ/yzFbHmrmeU/xcTzOj2tdXOp5y2JuKJQBnI3w6JJl9hXgb9m/0ok9vC7PqOL4qhSDD/pi4MGe1JZO3Zwa2WgF+s/YX8eaBjxDqiJIxfAdQsuZXrHlvyhwQNLVpiMcmwYh7S0MdQhgbUr9OeXDrUU5QaAlOQSGFSfRNH0oIPFoFpy5leatlvYhcjB9HtGoSgM06cYC6PivxA5m2jk9DYXv1+WgN6enkLwTJS9h+/I/bonmnv2DRNp1rXO+w9fHkObczsJNHIBmjTFPPUyYvtsXxREx0ZhaT5VcSfhbVF7IzDvAicUaK+zKwEnKC8Yx/thZ0rem3jUenrvJpG0wqc40dL2Jgds0r3NDXjRIA4CNuRXWuYGTwn9Pkqjjcy7kNDWqhYkWNQB8TPFJe53Bau2xE4R7l6k3gbXl7HQQVFKBAMVhJnDZy13Rz9rvk5L1hmlhU8RSvL7vWC+V0sFq+F2ZmpW5Wr/lpDeO5msbuBCvgIIx3M10M1Z2o+vOAevAZM= ceph-3ccd15e6-7804-11ef-baf6-efdc52797490 2024-09-21T10:29:34.236 INFO:teuthology.orchestra.run.smithi123.stderr:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-09-21T10:29:34.236 INFO:teuthology.orchestra.run.smithi123.stderr:Adding key to root@localhost authorized_keys... 2024-09-21T10:29:34.242 INFO:teuthology.orchestra.run.smithi123.stderr:Adding host smithi123... 2024-09-21T10:29:34.850 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:34 smithi123 ceph-mon[23709]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:29:35.729 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: Added host 'smithi123' with addr '172.21.15.123' 2024-09-21T10:29:36.317 INFO:teuthology.orchestra.run.smithi123.stderr:Deploying mon service with default placement... 2024-09-21T10:29:36.600 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:36 smithi123 ceph-mon[23709]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi123", "addr": "172.21.15.123", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:29:36.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:36 smithi123 ceph-mon[23709]: Deploying cephadm binary to smithi123 2024-09-21T10:29:36.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:36 smithi123 ceph-mon[23709]: from='mgr.14120 172.21.15.123:0/759259812' entity='mgr.smithi123.stnuit' 2024-09-21T10:29:36.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:36 smithi123 ceph-mon[23709]: from='mgr.14120 172.21.15.123:0/759259812' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:29:36.966 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: Scheduled mon update... 2024-09-21T10:29:37.850 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:37 smithi123 ceph-mon[23709]: Added host smithi123 2024-09-21T10:29:37.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:37 smithi123 ceph-mon[23709]: from='mgr.14120 172.21.15.123:0/759259812' entity='mgr.smithi123.stnuit' 2024-09-21T10:29:37.992 INFO:teuthology.orchestra.run.smithi123.stderr:Deploying mgr service with default placement... 2024-09-21T10:29:38.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:38 smithi123 ceph-mon[23709]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:29:38.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:38 smithi123 ceph-mon[23709]: Saving service mon spec with placement count:5 2024-09-21T10:29:39.499 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: Scheduled mgr update... 2024-09-21T10:29:40.071 INFO:teuthology.orchestra.run.smithi123.stderr:Deploying crash service with default placement... 2024-09-21T10:29:40.187 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:39 smithi123 ceph-mon[23709]: from='client.14144 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:29:40.187 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:39 smithi123 ceph-mon[23709]: Saving service mgr spec with placement count:2 2024-09-21T10:29:40.187 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:39 smithi123 ceph-mon[23709]: from='mgr.14120 172.21.15.123:0/759259812' entity='mgr.smithi123.stnuit' 2024-09-21T10:29:40.187 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:39 smithi123 ceph-mon[23709]: from='mgr.14120 172.21.15.123:0/759259812' entity='mgr.smithi123.stnuit' 2024-09-21T10:29:40.187 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:39 smithi123 ceph-mon[23709]: from='mgr.14120 172.21.15.123:0/759259812' entity='mgr.smithi123.stnuit' 2024-09-21T10:29:40.704 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: Scheduled crash update... 2024-09-21T10:29:41.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:41 smithi123 ceph-mon[23709]: from='client.14146 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "crash", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:29:41.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:41 smithi123 ceph-mon[23709]: Saving service crash spec with placement * 2024-09-21T10:29:41.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:41 smithi123 ceph-mon[23709]: from='mgr.14120 172.21.15.123:0/759259812' entity='mgr.smithi123.stnuit' 2024-09-21T10:29:41.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:41 smithi123 ceph-mon[23709]: from='mgr.14120 172.21.15.123:0/759259812' entity='mgr.smithi123.stnuit' 2024-09-21T10:29:42.008 INFO:teuthology.orchestra.run.smithi123.stderr:Deploying prometheus service with default placement... 2024-09-21T10:29:42.704 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: Scheduled prometheus update... 2024-09-21T10:29:43.384 INFO:teuthology.orchestra.run.smithi123.stderr:Deploying grafana service with default placement... 2024-09-21T10:29:43.743 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:43 smithi123 ceph-mon[23709]: from='mgr.14120 172.21.15.123:0/759259812' entity='mgr.smithi123.stnuit' 2024-09-21T10:29:43.743 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:43 smithi123 ceph-mon[23709]: from='mgr.14120 172.21.15.123:0/759259812' entity='mgr.smithi123.stnuit' 2024-09-21T10:29:44.038 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: Scheduled grafana update... 2024-09-21T10:29:44.571 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:44 smithi123 ceph-mon[23709]: from='client.14148 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:29:44.571 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:44 smithi123 ceph-mon[23709]: Saving service prometheus spec with placement count:1 2024-09-21T10:29:44.571 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:44 smithi123 ceph-mon[23709]: from='mgr.14120 172.21.15.123:0/759259812' entity='mgr.smithi123.stnuit' 2024-09-21T10:29:44.783 INFO:teuthology.orchestra.run.smithi123.stderr:Deploying node-exporter service with default placement... 2024-09-21T10:29:45.661 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:45 smithi123 ceph-mon[23709]: from='client.14150 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:29:45.661 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:45 smithi123 ceph-mon[23709]: Saving service grafana spec with placement count:1 2024-09-21T10:29:45.868 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: Scheduled node-exporter update... 2024-09-21T10:29:46.525 INFO:teuthology.orchestra.run.smithi123.stderr:Deploying alertmanager service with default placement... 2024-09-21T10:29:47.100 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:46 smithi123 ceph-mon[23709]: from='client.14152 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:29:47.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:46 smithi123 ceph-mon[23709]: Saving service node-exporter spec with placement * 2024-09-21T10:29:47.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:46 smithi123 ceph-mon[23709]: from='mgr.14120 172.21.15.123:0/759259812' entity='mgr.smithi123.stnuit' 2024-09-21T10:29:47.211 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: Scheduled alertmanager update... 2024-09-21T10:29:48.324 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:48 smithi123 ceph-mon[23709]: from='client.14154 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:29:48.324 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:48 smithi123 ceph-mon[23709]: Saving service alertmanager spec with placement count:1 2024-09-21T10:29:48.324 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:48 smithi123 ceph-mon[23709]: from='mgr.14120 172.21.15.123:0/759259812' entity='mgr.smithi123.stnuit' 2024-09-21T10:29:49.739 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:49 smithi123 ceph-mon[23709]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T10:29:49.739 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:49 smithi123 ceph-mon[23709]: from='client.? 172.21.15.123:0/4265273808' entity='client.admin' 2024-09-21T10:29:50.353 INFO:teuthology.orchestra.run.smithi123.stderr:Enabling the dashboard module... 2024-09-21T10:29:51.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:50 smithi123 ceph-mon[23709]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T10:29:51.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:50 smithi123 ceph-mon[23709]: from='client.? 172.21.15.123:0/1168173365' entity='client.admin' 2024-09-21T10:29:52.057 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:51 smithi123 ceph-mon[23709]: from='client.? 172.21.15.123:0/3993750027' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-09-21T10:29:52.922 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:52 smithi123 ceph-mon[23709]: from='client.? 172.21.15.123:0/3993750027' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-09-21T10:29:52.923 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:52 smithi123 ceph-mon[23709]: mgrmap e9: smithi123.stnuit(active, since 24s) 2024-09-21T10:29:53.085 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: { 2024-09-21T10:29:53.085 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "epoch": 9, 2024-09-21T10:29:53.085 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "available": true, 2024-09-21T10:29:53.085 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "active_name": "smithi123.stnuit", 2024-09-21T10:29:53.085 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "num_standby": 0 2024-09-21T10:29:53.085 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: } 2024-09-21T10:29:53.765 INFO:teuthology.orchestra.run.smithi123.stderr:Waiting for the mgr to restart... 2024-09-21T10:29:53.765 INFO:teuthology.orchestra.run.smithi123.stderr:Waiting for mgr epoch 9... 2024-09-21T10:29:54.018 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:53 smithi123 ceph-mon[23709]: from='client.? 172.21.15.123:0/3155034900' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-21T10:29:58.852 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: { 2024-09-21T10:29:58.852 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "mgrmap_epoch": 11, 2024-09-21T10:29:58.853 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: "initialized": true 2024-09-21T10:29:58.853 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: } 2024-09-21T10:29:59.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:58 smithi123 ceph-mon[23709]: Active manager daemon smithi123.stnuit restarted 2024-09-21T10:29:59.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:58 smithi123 ceph-mon[23709]: Activating manager daemon smithi123.stnuit 2024-09-21T10:29:59.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:58 smithi123 ceph-mon[23709]: osdmap e3: 0 total, 0 up, 0 in 2024-09-21T10:29:59.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:58 smithi123 ceph-mon[23709]: mgrmap e10: smithi123.stnuit(active, starting, since 0.0533071s) 2024-09-21T10:29:59.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:58 smithi123 ceph-mon[23709]: from='mgr.14162 172.21.15.123:0/3270503811' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mon metadata", "id": "smithi123"}]: dispatch 2024-09-21T10:29:59.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:58 smithi123 ceph-mon[23709]: from='mgr.14162 172.21.15.123:0/3270503811' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mgr metadata", "who": "smithi123.stnuit", "id": "smithi123.stnuit"}]: dispatch 2024-09-21T10:29:59.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:58 smithi123 ceph-mon[23709]: from='mgr.14162 172.21.15.123:0/3270503811' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T10:29:59.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:58 smithi123 ceph-mon[23709]: from='mgr.14162 172.21.15.123:0/3270503811' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T10:29:59.102 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:58 smithi123 ceph-mon[23709]: from='mgr.14162 172.21.15.123:0/3270503811' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T10:29:59.102 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:58 smithi123 ceph-mon[23709]: Manager daemon smithi123.stnuit is now available 2024-09-21T10:29:59.102 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:58 smithi123 ceph-mon[23709]: from='mgr.14162 172.21.15.123:0/3270503811' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:29:59.102 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:58 smithi123 ceph-mon[23709]: from='mgr.14162 172.21.15.123:0/3270503811' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:29:59.102 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:58 smithi123 ceph-mon[23709]: from='mgr.14162 172.21.15.123:0/3270503811' entity='mgr.smithi123.stnuit' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.stnuit/mirror_snapshot_schedule"}]: dispatch 2024-09-21T10:29:59.102 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:58 smithi123 ceph-mon[23709]: from='mgr.14162 172.21.15.123:0/3270503811' entity='mgr.smithi123.stnuit' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.stnuit/trash_purge_schedule"}]: dispatch 2024-09-21T10:29:59.102 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:58 smithi123 ceph-mon[23709]: [21/Sep/2024:10:29:58] ENGINE Bus STARTING 2024-09-21T10:29:59.102 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:58 smithi123 ceph-mon[23709]: [21/Sep/2024:10:29:58] ENGINE Serving on https://172.21.15.123:7150 2024-09-21T10:29:59.102 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:58 smithi123 ceph-mon[23709]: [21/Sep/2024:10:29:58] ENGINE Bus STARTED 2024-09-21T10:29:59.102 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:58 smithi123 ceph-mon[23709]: from='mgr.14162 172.21.15.123:0/3270503811' entity='mgr.smithi123.stnuit' 2024-09-21T10:29:59.516 INFO:teuthology.orchestra.run.smithi123.stderr:mgr epoch 9 is available 2024-09-21T10:29:59.517 INFO:teuthology.orchestra.run.smithi123.stderr:Generating a dashboard self-signed certificate... 2024-09-21T10:30:00.100 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:59 smithi123 ceph-mon[23709]: from='client.14166 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-09-21T10:30:00.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:59 smithi123 ceph-mon[23709]: mgrmap e11: smithi123.stnuit(active, since 1.05601s) 2024-09-21T10:30:00.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:29:59 smithi123 ceph-mon[23709]: from='client.14166 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-09-21T10:30:00.248 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: Self-signed certificate created 2024-09-21T10:30:00.857 INFO:teuthology.orchestra.run.smithi123.stderr:Creating initial admin user... 2024-09-21T10:30:01.100 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:00 smithi123 ceph-mon[23709]: mgrmap e12: smithi123.stnuit(active, since 2s) 2024-09-21T10:30:01.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:00 smithi123 ceph-mon[23709]: from='mgr.14162 172.21.15.123:0/3270503811' entity='mgr.smithi123.stnuit' 2024-09-21T10:30:01.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:00 smithi123 ceph-mon[23709]: from='client.14174 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:30:01.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:00 smithi123 ceph-mon[23709]: from='mgr.14162 172.21.15.123:0/3270503811' entity='mgr.smithi123.stnuit' 2024-09-21T10:30:01.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:00 smithi123 ceph-mon[23709]: from='mgr.14162 172.21.15.123:0/3270503811' entity='mgr.smithi123.stnuit' 2024-09-21T10:30:01.756 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: {"username": "admin", "password": "$2b$12$eEJ88J8uxTa6wBkHpqfMO.lKnoVCRJo2nZ8GZqy.W5rbBv.I.Ik0O", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1726914601, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-09-21T10:30:02.412 INFO:teuthology.orchestra.run.smithi123.stderr:Fetching dashboard port number... 2024-09-21T10:30:02.687 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:02 smithi123 ceph-mon[23709]: from='mgr.14162 172.21.15.123:0/3270503811' entity='mgr.smithi123.stnuit' 2024-09-21T10:30:02.687 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:02 smithi123 ceph-mon[23709]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:30:02.687 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:02 smithi123 ceph-mon[23709]: from='mgr.14162 172.21.15.123:0/3270503811' entity='mgr.smithi123.stnuit' 2024-09-21T10:30:03.188 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: 8443 2024-09-21T10:30:03.600 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:03 smithi123 ceph-mon[23709]: from='client.? 172.21.15.123:0/4259848990' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-09-21T10:30:03.809 INFO:teuthology.orchestra.run.smithi123.stderr:systemctl: disabled 2024-09-21T10:30:03.815 INFO:teuthology.orchestra.run.smithi123.stderr:systemctl: inactive 2024-09-21T10:30:03.816 INFO:teuthology.orchestra.run.smithi123.stderr:firewalld.service is not enabled 2024-09-21T10:30:03.816 INFO:teuthology.orchestra.run.smithi123.stderr:Not possible to open ports <[8443]>. firewalld.service is not available 2024-09-21T10:30:03.818 INFO:teuthology.orchestra.run.smithi123.stderr:Ceph Dashboard is now available at: 2024-09-21T10:30:03.818 INFO:teuthology.orchestra.run.smithi123.stderr: 2024-09-21T10:30:03.818 INFO:teuthology.orchestra.run.smithi123.stderr: URL: https://smithi123.front.sepia.ceph.com:8443/ 2024-09-21T10:30:03.818 INFO:teuthology.orchestra.run.smithi123.stderr: User: admin 2024-09-21T10:30:03.818 INFO:teuthology.orchestra.run.smithi123.stderr: Password: bdt8wp5mbu 2024-09-21T10:30:03.818 INFO:teuthology.orchestra.run.smithi123.stderr: 2024-09-21T10:30:03.819 INFO:teuthology.orchestra.run.smithi123.stderr:Enabling autotune for osd_memory_target 2024-09-21T10:30:05.656 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:05 smithi123 ceph-mon[23709]: mgrmap e13: smithi123.stnuit(active, since 6s) 2024-09-21T10:30:06.146 INFO:teuthology.orchestra.run.smithi123.stderr:/usr/bin/ceph: set mgr/dashboard/cluster/status 2024-09-21T10:30:06.756 INFO:teuthology.orchestra.run.smithi123.stderr:You can access the Ceph CLI with: 2024-09-21T10:30:06.757 INFO:teuthology.orchestra.run.smithi123.stderr: 2024-09-21T10:30:06.757 INFO:teuthology.orchestra.run.smithi123.stderr: sudo /home/ubuntu/cephtest/cephadm shell --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-09-21T10:30:06.757 INFO:teuthology.orchestra.run.smithi123.stderr: 2024-09-21T10:30:06.757 INFO:teuthology.orchestra.run.smithi123.stderr:Please consider enabling telemetry to help improve Ceph: 2024-09-21T10:30:06.757 INFO:teuthology.orchestra.run.smithi123.stderr: 2024-09-21T10:30:06.757 INFO:teuthology.orchestra.run.smithi123.stderr: ceph telemetry on 2024-09-21T10:30:06.757 INFO:teuthology.orchestra.run.smithi123.stderr: 2024-09-21T10:30:06.757 INFO:teuthology.orchestra.run.smithi123.stderr:For more information see: 2024-09-21T10:30:06.757 INFO:teuthology.orchestra.run.smithi123.stderr: 2024-09-21T10:30:06.757 INFO:teuthology.orchestra.run.smithi123.stderr: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-09-21T10:30:06.757 INFO:teuthology.orchestra.run.smithi123.stderr: 2024-09-21T10:30:06.757 INFO:teuthology.orchestra.run.smithi123.stderr:Bootstrap complete. 2024-09-21T10:30:06.792 INFO:tasks.cephadm:Fetching config... 2024-09-21T10:30:06.792 DEBUG:teuthology.orchestra.run.smithi123:> set -ex 2024-09-21T10:30:06.792 DEBUG:teuthology.orchestra.run.smithi123:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-09-21T10:30:06.808 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-09-21T10:30:06.808 DEBUG:teuthology.orchestra.run.smithi123:> set -ex 2024-09-21T10:30:06.808 DEBUG:teuthology.orchestra.run.smithi123:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-09-21T10:30:06.864 INFO:tasks.cephadm:Fetching mon keyring... 2024-09-21T10:30:06.864 DEBUG:teuthology.orchestra.run.smithi123:> set -ex 2024-09-21T10:30:06.864 DEBUG:teuthology.orchestra.run.smithi123:> sudo dd if=/var/lib/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/mon.smithi123/keyring of=/dev/stdout 2024-09-21T10:30:06.929 INFO:tasks.cephadm:Fetching pub ssh key... 2024-09-21T10:30:06.929 DEBUG:teuthology.orchestra.run.smithi123:> set -ex 2024-09-21T10:30:06.929 DEBUG:teuthology.orchestra.run.smithi123:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-09-21T10:30:06.985 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-09-21T10:30:06.985 DEBUG:teuthology.orchestra.run.smithi123:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDZ/yzFbHmrmeU/xcTzOj2tdXOp5y2JuKJQBnI3w6JJl9hXgb9m/0ok9vC7PqOL4qhSDD/pi4MGe1JZO3Zwa2WgF+s/YX8eaBjxDqiJIxfAdQsuZXrHlvyhwQNLVpiMcmwYh7S0MdQhgbUr9OeXDrUU5QaAlOQSGFSfRNH0oIPFoFpy5leatlvYhcjB9HtGoSgM06cYC6PivxA5m2jk9DYXv1+WgN6enkLwTJS9h+/I/bonmnv2DRNp1rXO+w9fHkObczsJNHIBmjTFPPUyYvtsXxREx0ZhaT5VcSfhbVF7IzDvAicUaK+zKwEnKC8Yx/thZ0rem3jUenrvJpG0wqc40dL2Jgds0r3NDXjRIA4CNuRXWuYGTwn9Pkqjjcy7kNDWqhYkWNQB8TPFJe53Bau2xE4R7l6k3gbXl7HQQVFKBAMVhJnDZy13Rz9rvk5L1hmlhU8RSvL7vWC+V0sFq+F2ZmpW5Wr/lpDeO5msbuBCvgIIx3M10M1Z2o+vOAevAZM= ceph-3ccd15e6-7804-11ef-baf6-efdc52797490' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-09-21T10:30:07.058 INFO:teuthology.orchestra.run.smithi123.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDZ/yzFbHmrmeU/xcTzOj2tdXOp5y2JuKJQBnI3w6JJl9hXgb9m/0ok9vC7PqOL4qhSDD/pi4MGe1JZO3Zwa2WgF+s/YX8eaBjxDqiJIxfAdQsuZXrHlvyhwQNLVpiMcmwYh7S0MdQhgbUr9OeXDrUU5QaAlOQSGFSfRNH0oIPFoFpy5leatlvYhcjB9HtGoSgM06cYC6PivxA5m2jk9DYXv1+WgN6enkLwTJS9h+/I/bonmnv2DRNp1rXO+w9fHkObczsJNHIBmjTFPPUyYvtsXxREx0ZhaT5VcSfhbVF7IzDvAicUaK+zKwEnKC8Yx/thZ0rem3jUenrvJpG0wqc40dL2Jgds0r3NDXjRIA4CNuRXWuYGTwn9Pkqjjcy7kNDWqhYkWNQB8TPFJe53Bau2xE4R7l6k3gbXl7HQQVFKBAMVhJnDZy13Rz9rvk5L1hmlhU8RSvL7vWC+V0sFq+F2ZmpW5Wr/lpDeO5msbuBCvgIIx3M10M1Z2o+vOAevAZM= ceph-3ccd15e6-7804-11ef-baf6-efdc52797490 2024-09-21T10:30:07.069 DEBUG:teuthology.orchestra.run.smithi175:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDZ/yzFbHmrmeU/xcTzOj2tdXOp5y2JuKJQBnI3w6JJl9hXgb9m/0ok9vC7PqOL4qhSDD/pi4MGe1JZO3Zwa2WgF+s/YX8eaBjxDqiJIxfAdQsuZXrHlvyhwQNLVpiMcmwYh7S0MdQhgbUr9OeXDrUU5QaAlOQSGFSfRNH0oIPFoFpy5leatlvYhcjB9HtGoSgM06cYC6PivxA5m2jk9DYXv1+WgN6enkLwTJS9h+/I/bonmnv2DRNp1rXO+w9fHkObczsJNHIBmjTFPPUyYvtsXxREx0ZhaT5VcSfhbVF7IzDvAicUaK+zKwEnKC8Yx/thZ0rem3jUenrvJpG0wqc40dL2Jgds0r3NDXjRIA4CNuRXWuYGTwn9Pkqjjcy7kNDWqhYkWNQB8TPFJe53Bau2xE4R7l6k3gbXl7HQQVFKBAMVhJnDZy13Rz9rvk5L1hmlhU8RSvL7vWC+V0sFq+F2ZmpW5Wr/lpDeO5msbuBCvgIIx3M10M1Z2o+vOAevAZM= ceph-3ccd15e6-7804-11ef-baf6-efdc52797490' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-09-21T10:30:07.332 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:07 smithi123 ceph-mon[23709]: from='client.? 172.21.15.123:0/4289202741' entity='client.admin' 2024-09-21T10:30:07.353 INFO:teuthology.orchestra.run.smithi175.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDZ/yzFbHmrmeU/xcTzOj2tdXOp5y2JuKJQBnI3w6JJl9hXgb9m/0ok9vC7PqOL4qhSDD/pi4MGe1JZO3Zwa2WgF+s/YX8eaBjxDqiJIxfAdQsuZXrHlvyhwQNLVpiMcmwYh7S0MdQhgbUr9OeXDrUU5QaAlOQSGFSfRNH0oIPFoFpy5leatlvYhcjB9HtGoSgM06cYC6PivxA5m2jk9DYXv1+WgN6enkLwTJS9h+/I/bonmnv2DRNp1rXO+w9fHkObczsJNHIBmjTFPPUyYvtsXxREx0ZhaT5VcSfhbVF7IzDvAicUaK+zKwEnKC8Yx/thZ0rem3jUenrvJpG0wqc40dL2Jgds0r3NDXjRIA4CNuRXWuYGTwn9Pkqjjcy7kNDWqhYkWNQB8TPFJe53Bau2xE4R7l6k3gbXl7HQQVFKBAMVhJnDZy13Rz9rvk5L1hmlhU8RSvL7vWC+V0sFq+F2ZmpW5Wr/lpDeO5msbuBCvgIIx3M10M1Z2o+vOAevAZM= ceph-3ccd15e6-7804-11ef-baf6-efdc52797490 2024-09-21T10:30:07.366 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-09-21T10:30:10.088 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:10 smithi123 ceph-mon[23709]: from='mgr.14162 172.21.15.123:0/3270503811' entity='mgr.smithi123.stnuit' 2024-09-21T10:30:10.088 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:10 smithi123 ceph-mon[23709]: from='mgr.14162 172.21.15.123:0/3270503811' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config rm", "who": "osd/host:smithi123", "name": "osd_memory_target"}]: dispatch 2024-09-21T10:30:10.088 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:10 smithi123 ceph-mon[23709]: from='mgr.14162 172.21.15.123:0/3270503811' entity='mgr.smithi123.stnuit' 2024-09-21T10:30:10.088 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:10 smithi123 ceph-mon[23709]: Deploying daemon alertmanager.smithi123 on smithi123 2024-09-21T10:30:11.043 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-09-21T10:30:11.044 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-09-21T10:30:11.472 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:11 smithi123 ceph-mon[23709]: from='client.? 172.21.15.123:0/2002904069' entity='client.admin' 2024-09-21T10:30:14.848 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi175 2024-09-21T10:30:14.848 DEBUG:teuthology.orchestra.run.smithi175:> set -ex 2024-09-21T10:30:14.848 DEBUG:teuthology.orchestra.run.smithi175:> dd of=/etc/ceph/ceph.conf 2024-09-21T10:30:14.868 DEBUG:teuthology.orchestra.run.smithi175:> set -ex 2024-09-21T10:30:14.868 DEBUG:teuthology.orchestra.run.smithi175:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-21T10:30:14.928 INFO:tasks.cephadm:Adding host smithi175 to orchestrator... 2024-09-21T10:30:14.928 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -- ceph orch host add smithi175 2024-09-21T10:30:14.951 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:14 smithi123 ceph-mon[23709]: from='client.14186 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:30:14.951 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:14 smithi123 ceph-mon[23709]: from='mgr.14162 172.21.15.123:0/3270503811' entity='mgr.smithi123.stnuit' 2024-09-21T10:30:19.037 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:18 smithi123 ceph-mon[23709]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T10:30:20.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:19 smithi123 ceph-mon[23709]: from='client.14188 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi175", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:30:20.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:19 smithi123 ceph-mon[23709]: Deploying cephadm binary to smithi175 2024-09-21T10:30:20.181 INFO:teuthology.orchestra.run.smithi123.stdout:Added host 'smithi175' with addr '172.21.15.175' 2024-09-21T10:30:21.184 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -- ceph orch host ls --format=json 2024-09-21T10:30:21.417 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:21 smithi123 ceph-mon[23709]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T10:30:21.417 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:21 smithi123 ceph-mon[23709]: from='mgr.14162 172.21.15.123:0/3270503811' entity='mgr.smithi123.stnuit' 2024-09-21T10:30:21.417 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:21 smithi123 ceph-mon[23709]: Added host smithi175 2024-09-21T10:30:23.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:22 smithi123 ceph-mon[23709]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T10:30:23.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:22 smithi123 ceph-mon[23709]: from='mgr.14162 172.21.15.123:0/3270503811' entity='mgr.smithi123.stnuit' 2024-09-21T10:30:23.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:22 smithi123 ceph-mon[23709]: from='mgr.14162 172.21.15.123:0/3270503811' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi123", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-21T10:30:23.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:22 smithi123 ceph-mon[23709]: from='mgr.14162 172.21.15.123:0/3270503811' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi123", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-09-21T10:30:23.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:22 smithi123 ceph-mon[23709]: from='mgr.14162 172.21.15.123:0/3270503811' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:30:23.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:22 smithi123 ceph-mon[23709]: Deploying daemon crash.smithi123 on smithi123 2024-09-21T10:30:23.602 INFO:teuthology.orchestra.run.smithi123.stdout: 2024-09-21T10:30:23.603 INFO:teuthology.orchestra.run.smithi123.stdout:[{"addr": "172.21.15.123", "hostname": "smithi123", "labels": [], "status": ""}, {"addr": "172.21.15.175", "hostname": "smithi175", "labels": [], "status": ""}] 2024-09-21T10:30:23.967 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:23 smithi123 ceph-mon[23709]: from='mgr.14162 172.21.15.123:0/3270503811' entity='mgr.smithi123.stnuit' 2024-09-21T10:30:23.967 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:23 smithi123 ceph-mon[23709]: from='client.14190 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:30:25.166 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:24 smithi123 ceph-mon[23709]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T10:30:25.193 INFO:tasks.cephadm:Setting crush tunables to default 2024-09-21T10:30:25.193 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -- ceph osd crush tunables default 2024-09-21T10:30:27.068 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:26 smithi123 ceph-mon[23709]: from='mgr.14162 172.21.15.123:0/3270503811' entity='mgr.smithi123.stnuit' 2024-09-21T10:30:27.068 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:26 smithi123 ceph-mon[23709]: from='mgr.14162 172.21.15.123:0/3270503811' entity='mgr.smithi123.stnuit' 2024-09-21T10:30:27.068 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:26 smithi123 ceph-mon[23709]: from='mgr.14162 172.21.15.123:0/3270503811' entity='mgr.smithi123.stnuit' 2024-09-21T10:30:27.068 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:26 smithi123 ceph-mon[23709]: from='mgr.14162 172.21.15.123:0/3270503811' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-21T10:30:27.068 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:26 smithi123 ceph-mon[23709]: from='mgr.14162 172.21.15.123:0/3270503811' entity='mgr.smithi123.stnuit' 2024-09-21T10:30:27.853 INFO:teuthology.orchestra.run.smithi123.stderr:adjusted tunables profile to default 2024-09-21T10:30:28.100 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:27 smithi123 ceph-mon[23709]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T10:30:28.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:27 smithi123 ceph-mon[23709]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-21T10:30:28.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:27 smithi123 ceph-mon[23709]: Deploying daemon grafana.smithi123 on smithi123 2024-09-21T10:30:28.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:27 smithi123 ceph-mon[23709]: from='client.? 172.21.15.123:0/2719487495' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-09-21T10:30:28.871 INFO:tasks.cephadm:Adding mon.smithi123 on smithi123 2024-09-21T10:30:28.872 INFO:tasks.cephadm:Adding mon.smithi175 on smithi175 2024-09-21T10:30:28.872 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -- ceph orch apply mon '2;smithi123:172.21.15.123=smithi123;smithi175:172.21.15.175=smithi175' 2024-09-21T10:30:29.100 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:28 smithi123 ceph-mon[23709]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T10:30:29.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:28 smithi123 ceph-mon[23709]: from='client.? 172.21.15.123:0/2719487495' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-09-21T10:30:29.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:28 smithi123 ceph-mon[23709]: osdmap e4: 0 total, 0 up, 0 in 2024-09-21T10:30:29.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:28 smithi123 ceph-mon[23709]: from='mgr.14162 172.21.15.123:0/3270503811' entity='mgr.smithi123.stnuit' 2024-09-21T10:30:30.803 INFO:teuthology.orchestra.run.smithi175.stdout:Scheduled mon update... 2024-09-21T10:30:31.274 DEBUG:teuthology.orchestra.run.smithi175:mon.smithi175> sudo journalctl -f -n 0 -u ceph-3ccd15e6-7804-11ef-baf6-efdc52797490@mon.smithi175.service 2024-09-21T10:30:31.276 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-21T10:30:31.276 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -- ceph mon dump -f json 2024-09-21T10:30:31.850 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:31 smithi123 ceph-mon[23709]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T10:30:31.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:31 smithi123 ceph-mon[23709]: from='mgr.14162 172.21.15.123:0/3270503811' entity='mgr.smithi123.stnuit' 2024-09-21T10:30:32.850 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:32 smithi123 ceph-mon[23709]: from='client.14194 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "2;smithi123:172.21.15.123=smithi123;smithi175:172.21.15.175=smithi175", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:30:32.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:32 smithi123 ceph-mon[23709]: Saving service mon spec with placement smithi123:172.21.15.123=smithi123;smithi175:172.21.15.175=smithi175;count:2 2024-09-21T10:30:33.028 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-09-21T10:30:33.028 INFO:teuthology.orchestra.run.smithi175.stdout:{"epoch":1,"fsid":"3ccd15e6-7804-11ef-baf6-efdc52797490","modified":"2024-09-21T10:28:48.510560Z","created":"2024-09-21T10:28:48.510560Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi123","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:3300","nonce":0},{"type":"v1","addr":"172.21.15.123:6789","nonce":0}]},"addr":"172.21.15.123:6789/0","public_addr":"172.21.15.123:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-21T10:30:33.030 INFO:teuthology.orchestra.run.smithi175.stderr:dumped monmap epoch 1 2024-09-21T10:30:33.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:33 smithi123 ceph-mon[23709]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T10:30:33.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:33 smithi123 ceph-mon[23709]: from='client.? 172.21.15.175:0/4034499394' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T10:30:34.532 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-21T10:30:34.532 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -- ceph mon dump -f json 2024-09-21T10:30:35.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:35 smithi123 ceph-mon[23709]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T10:30:36.239 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-09-21T10:30:36.239 INFO:teuthology.orchestra.run.smithi175.stdout:{"epoch":1,"fsid":"3ccd15e6-7804-11ef-baf6-efdc52797490","modified":"2024-09-21T10:28:48.510560Z","created":"2024-09-21T10:28:48.510560Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi123","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:3300","nonce":0},{"type":"v1","addr":"172.21.15.123:6789","nonce":0}]},"addr":"172.21.15.123:6789/0","public_addr":"172.21.15.123:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-21T10:30:36.241 INFO:teuthology.orchestra.run.smithi175.stderr:dumped monmap epoch 1 2024-09-21T10:30:36.849 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:36 smithi123 ceph-mon[23709]: from='client.? 172.21.15.175:0/791042265' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T10:30:37.607 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:37 smithi123 ceph-mon[23709]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T10:30:37.607 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:37 smithi123 ceph-mon[23709]: from='mgr.14162 172.21.15.123:0/3270503811' entity='mgr.smithi123.stnuit' 2024-09-21T10:30:37.848 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-21T10:30:37.849 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -- ceph mon dump -f json 2024-09-21T10:30:38.850 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:38 smithi123 ceph-mon[23709]: Deploying daemon node-exporter.smithi123 on smithi123 2024-09-21T10:30:38.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:38 smithi123 ceph-mon[23709]: from='mgr.14162 172.21.15.123:0/3270503811' entity='mgr.smithi123.stnuit' 2024-09-21T10:30:39.535 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-09-21T10:30:39.535 INFO:teuthology.orchestra.run.smithi175.stdout:{"epoch":1,"fsid":"3ccd15e6-7804-11ef-baf6-efdc52797490","modified":"2024-09-21T10:28:48.510560Z","created":"2024-09-21T10:28:48.510560Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi123","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:3300","nonce":0},{"type":"v1","addr":"172.21.15.123:6789","nonce":0}]},"addr":"172.21.15.123:6789/0","public_addr":"172.21.15.123:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-21T10:30:39.537 INFO:teuthology.orchestra.run.smithi175.stderr:dumped monmap epoch 1 2024-09-21T10:30:39.846 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:39 smithi123 ceph-mon[23709]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T10:30:39.846 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:39 smithi123 ceph-mon[23709]: from='client.? 172.21.15.175:0/3847486747' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T10:30:41.091 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-21T10:30:41.092 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -- ceph mon dump -f json 2024-09-21T10:30:41.578 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:41 smithi123 ceph-mon[23709]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T10:30:41.578 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:41 smithi123 ceph-mon[23709]: from='mgr.14162 172.21.15.123:0/3270503811' entity='mgr.smithi123.stnuit' 2024-09-21T10:30:41.578 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:41 smithi123 ceph-mon[23709]: from='mgr.14162 172.21.15.123:0/3270503811' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-21T10:30:42.328 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:42 smithi123 ceph-mon[23709]: from='mgr.14162 172.21.15.123:0/3270503811' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-21T10:30:42.328 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:42 smithi123 ceph-mon[23709]: mgrmap e14: smithi123.stnuit(active, since 43s) 2024-09-21T10:30:42.736 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-09-21T10:30:42.736 INFO:teuthology.orchestra.run.smithi175.stdout:{"epoch":1,"fsid":"3ccd15e6-7804-11ef-baf6-efdc52797490","modified":"2024-09-21T10:28:48.510560Z","created":"2024-09-21T10:28:48.510560Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi123","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:3300","nonce":0},{"type":"v1","addr":"172.21.15.123:6789","nonce":0}]},"addr":"172.21.15.123:6789/0","public_addr":"172.21.15.123:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-21T10:30:42.738 INFO:teuthology.orchestra.run.smithi175.stderr:dumped monmap epoch 1 2024-09-21T10:30:43.600 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:43 smithi123 ceph-mon[23709]: from='client.? 172.21.15.175:0/1235557660' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T10:30:44.226 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-21T10:30:44.226 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -- ceph mon dump -f json 2024-09-21T10:30:45.988 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-09-21T10:30:45.988 INFO:teuthology.orchestra.run.smithi175.stdout:{"epoch":1,"fsid":"3ccd15e6-7804-11ef-baf6-efdc52797490","modified":"2024-09-21T10:28:48.510560Z","created":"2024-09-21T10:28:48.510560Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi123","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:3300","nonce":0},{"type":"v1","addr":"172.21.15.123:6789","nonce":0}]},"addr":"172.21.15.123:6789/0","public_addr":"172.21.15.123:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-21T10:30:45.990 INFO:teuthology.orchestra.run.smithi175.stderr:dumped monmap epoch 1 2024-09-21T10:30:46.850 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:46 smithi123 ceph-mon[23709]: from='client.? 172.21.15.175:0/98689285' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T10:30:47.571 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-21T10:30:47.572 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -- ceph mon dump -f json 2024-09-21T10:30:47.835 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:47 smithi123 ceph-mon[23709]: Active manager daemon smithi123.stnuit restarted 2024-09-21T10:30:47.835 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:47 smithi123 ceph-mon[23709]: Activating manager daemon smithi123.stnuit 2024-09-21T10:30:47.835 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:47 smithi123 ceph-mon[23709]: osdmap e5: 0 total, 0 up, 0 in 2024-09-21T10:30:48.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:48 smithi123 ceph-mon[23709]: mgrmap e15: smithi123.stnuit(active, starting, since 0.401794s) 2024-09-21T10:30:48.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:48 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mon metadata", "id": "smithi123"}]: dispatch 2024-09-21T10:30:48.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:48 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mgr metadata", "who": "smithi123.stnuit", "id": "smithi123.stnuit"}]: dispatch 2024-09-21T10:30:48.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:48 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T10:30:48.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:48 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T10:30:48.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:48 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T10:30:48.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:48 smithi123 ceph-mon[23709]: Manager daemon smithi123.stnuit is now available 2024-09-21T10:30:48.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:48 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:30:48.852 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:48 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:30:48.852 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:48 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:30:48.852 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:48 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T10:30:48.852 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:48 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:30:48.852 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:48 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.stnuit/mirror_snapshot_schedule"}]: dispatch 2024-09-21T10:30:48.852 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:48 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.stnuit/trash_purge_schedule"}]: dispatch 2024-09-21T10:30:48.852 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:48 smithi123 ceph-mon[23709]: [21/Sep/2024:10:30:47] ENGINE Bus STARTING 2024-09-21T10:30:48.852 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:48 smithi123 ceph-mon[23709]: [21/Sep/2024:10:30:47] ENGINE Serving on https://172.21.15.123:7150 2024-09-21T10:30:48.852 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:48 smithi123 ceph-mon[23709]: [21/Sep/2024:10:30:47] ENGINE Bus STARTED 2024-09-21T10:30:48.852 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:48 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:30:48.852 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:48 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:30:48.852 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:48 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:30:49.596 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-09-21T10:30:49.596 INFO:teuthology.orchestra.run.smithi175.stdout:{"epoch":1,"fsid":"3ccd15e6-7804-11ef-baf6-efdc52797490","modified":"2024-09-21T10:28:48.510560Z","created":"2024-09-21T10:28:48.510560Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi123","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:3300","nonce":0},{"type":"v1","addr":"172.21.15.123:6789","nonce":0}]},"addr":"172.21.15.123:6789/0","public_addr":"172.21.15.123:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-21T10:30:49.598 INFO:teuthology.orchestra.run.smithi175.stderr:dumped monmap epoch 1 2024-09-21T10:30:49.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:49 smithi123 ceph-mon[23709]: mgrmap e16: smithi123.stnuit(active, since 1.40751s) 2024-09-21T10:30:50.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:50 smithi123 ceph-mon[23709]: mgrmap e17: smithi123.stnuit(active, since 2s) 2024-09-21T10:30:50.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:50 smithi123 ceph-mon[23709]: from='client.? 172.21.15.175:0/911067418' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T10:30:50.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:50 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:30:51.163 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-21T10:30:51.163 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -- ceph mon dump -f json 2024-09-21T10:30:54.123 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-09-21T10:30:54.123 INFO:teuthology.orchestra.run.smithi175.stdout:{"epoch":1,"fsid":"3ccd15e6-7804-11ef-baf6-efdc52797490","modified":"2024-09-21T10:28:48.510560Z","created":"2024-09-21T10:28:48.510560Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi123","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:3300","nonce":0},{"type":"v1","addr":"172.21.15.123:6789","nonce":0}]},"addr":"172.21.15.123:6789/0","public_addr":"172.21.15.123:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-21T10:30:54.125 INFO:teuthology.orchestra.run.smithi175.stderr:dumped monmap epoch 1 2024-09-21T10:30:54.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:54 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:30:54.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:54 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:30:54.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:54 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config rm", "who": "osd/host:smithi175", "name": "osd_memory_target"}]: dispatch 2024-09-21T10:30:54.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:54 smithi123 ceph-mon[23709]: from='client.? 172.21.15.175:0/2527966737' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T10:30:55.671 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-21T10:30:55.671 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -- ceph mon dump -f json 2024-09-21T10:30:56.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:56 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:30:56.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:56 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:30:56.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:56 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config rm", "who": "osd/host:smithi123", "name": "osd_memory_target"}]: dispatch 2024-09-21T10:30:56.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:56 smithi123 ceph-mon[23709]: Updating smithi123:/etc/ceph/ceph.conf 2024-09-21T10:30:56.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:56 smithi123 ceph-mon[23709]: Updating smithi123:/etc/ceph/ceph.client.admin.keyring 2024-09-21T10:30:56.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:56 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:30:56.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:56 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:30:56.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:56 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi175", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-21T10:30:56.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:56 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi175", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-09-21T10:30:56.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:56 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:30:57.685 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-09-21T10:30:57.685 INFO:teuthology.orchestra.run.smithi175.stdout:{"epoch":1,"fsid":"3ccd15e6-7804-11ef-baf6-efdc52797490","modified":"2024-09-21T10:28:48.510560Z","created":"2024-09-21T10:28:48.510560Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi123","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:3300","nonce":0},{"type":"v1","addr":"172.21.15.123:6789","nonce":0}]},"addr":"172.21.15.123:6789/0","public_addr":"172.21.15.123:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-21T10:30:57.687 INFO:teuthology.orchestra.run.smithi175.stderr:dumped monmap epoch 1 2024-09-21T10:30:57.850 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:57 smithi123 ceph-mon[23709]: Deploying daemon crash.smithi175 on smithi175 2024-09-21T10:30:58.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:30:58 smithi123 ceph-mon[23709]: from='client.? 172.21.15.175:0/559409351' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T10:30:59.377 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-21T10:30:59.378 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -- ceph mon dump -f json 2024-09-21T10:31:01.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:01 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:31:01.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:01 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi175.txvwrq", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T10:31:01.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:01 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.smithi175.txvwrq", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-21T10:31:01.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:01 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T10:31:01.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:01 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:31:01.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:01 smithi123 ceph-mon[23709]: Deploying daemon mgr.smithi175.txvwrq on smithi175 2024-09-21T10:31:02.053 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-09-21T10:31:02.054 INFO:teuthology.orchestra.run.smithi175.stdout:{"epoch":1,"fsid":"3ccd15e6-7804-11ef-baf6-efdc52797490","modified":"2024-09-21T10:28:48.510560Z","created":"2024-09-21T10:28:48.510560Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi123","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:3300","nonce":0},{"type":"v1","addr":"172.21.15.123:6789","nonce":0}]},"addr":"172.21.15.123:6789/0","public_addr":"172.21.15.123:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-21T10:31:02.056 INFO:teuthology.orchestra.run.smithi175.stderr:dumped monmap epoch 1 2024-09-21T10:31:02.850 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:02 smithi123 ceph-mon[23709]: from='client.? 172.21.15.175:0/1266091902' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T10:31:04.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:03 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:31:04.412 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-21T10:31:04.412 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -- ceph mon dump -f json 2024-09-21T10:31:05.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:04 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:31:05.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:04 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T10:31:05.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:04 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:31:05.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:04 smithi123 ceph-mon[23709]: Deploying daemon mon.smithi175 on smithi175 2024-09-21T10:31:08.036 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:07 smithi175 systemd[1]: Started Ceph mon.smithi175 for 3ccd15e6-7804-11ef-baf6-efdc52797490. 2024-09-21T10:31:12.958 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-09-21T10:31:12.958 INFO:teuthology.orchestra.run.smithi175.stdout:{"epoch":2,"fsid":"3ccd15e6-7804-11ef-baf6-efdc52797490","modified":"2024-09-21T10:31:07.943649Z","created":"2024-09-21T10:28:48.510560Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi123","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:3300","nonce":0},{"type":"v1","addr":"172.21.15.123:6789","nonce":0}]},"addr":"172.21.15.123:6789/0","public_addr":"172.21.15.123:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"smithi175","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:3300","nonce":0},{"type":"v1","addr":"172.21.15.175:6789","nonce":0}]},"addr":"172.21.15.175:6789/0","public_addr":"172.21.15.175:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-09-21T10:31:12.961 INFO:teuthology.orchestra.run.smithi175.stderr:dumped monmap epoch 2 2024-09-21T10:31:13.336 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:13 smithi175 ceph-mon[25806]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T10:31:13.336 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:13 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mon metadata", "id": "smithi123"}]: dispatch 2024-09-21T10:31:13.336 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:13 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mon metadata", "id": "smithi175"}]: dispatch 2024-09-21T10:31:13.336 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:13 smithi175 ceph-mon[25806]: mon.smithi123 calling monitor election 2024-09-21T10:31:13.336 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:13 smithi175 ceph-mon[25806]: from='client.? 172.21.15.175:0/44878302' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T10:31:13.336 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:13 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mon metadata", "id": "smithi175"}]: dispatch 2024-09-21T10:31:13.336 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:13 smithi175 ceph-mon[25806]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T10:31:13.336 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:13 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mon metadata", "id": "smithi175"}]: dispatch 2024-09-21T10:31:13.336 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:13 smithi175 ceph-mon[25806]: mon.smithi175 calling monitor election 2024-09-21T10:31:13.337 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:13 smithi175 ceph-mon[25806]: from='mgr.? 172.21.15.175:0/2910286930' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi175.txvwrq/crt"}]: dispatch 2024-09-21T10:31:13.337 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:13 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mon metadata", "id": "smithi175"}]: dispatch 2024-09-21T10:31:13.337 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:13 smithi175 ceph-mon[25806]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T10:31:13.337 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:13 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mon metadata", "id": "smithi175"}]: dispatch 2024-09-21T10:31:13.337 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:13 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mon metadata", "id": "smithi175"}]: dispatch 2024-09-21T10:31:13.337 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:13 smithi175 ceph-mon[25806]: mon.smithi123 is new leader, mons smithi123,smithi175 in quorum (ranks 0,1) 2024-09-21T10:31:13.337 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:13 smithi175 ceph-mon[25806]: monmap e2: 2 mons at {smithi123=[v2:172.21.15.123:3300/0,v1:172.21.15.123:6789/0],smithi175=[v2:172.21.15.175:3300/0,v1:172.21.15.175:6789/0]} 2024-09-21T10:31:13.337 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:13 smithi175 ceph-mon[25806]: fsmap 2024-09-21T10:31:13.337 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:13 smithi175 ceph-mon[25806]: osdmap e5: 0 total, 0 up, 0 in 2024-09-21T10:31:13.337 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:13 smithi175 ceph-mon[25806]: mgrmap e17: smithi123.stnuit(active, since 25s) 2024-09-21T10:31:13.337 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:13 smithi175 ceph-mon[25806]: Standby manager daemon smithi175.txvwrq started 2024-09-21T10:31:13.337 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:13 smithi175 ceph-mon[25806]: from='mgr.? 172.21.15.175:0/2910286930' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T10:31:13.337 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:13 smithi175 ceph-mon[25806]: overall HEALTH_OK 2024-09-21T10:31:13.337 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:13 smithi175 ceph-mon[25806]: from='mgr.? 172.21.15.175:0/2910286930' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi175.txvwrq/key"}]: dispatch 2024-09-21T10:31:13.337 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:13 smithi175 ceph-mon[25806]: from='mgr.? 172.21.15.175:0/2910286930' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T10:31:13.338 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:13 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:31:13.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:13 smithi123 ceph-mon[23709]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T10:31:13.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:13 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mon metadata", "id": "smithi123"}]: dispatch 2024-09-21T10:31:13.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:13 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mon metadata", "id": "smithi175"}]: dispatch 2024-09-21T10:31:13.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:13 smithi123 ceph-mon[23709]: mon.smithi123 calling monitor election 2024-09-21T10:31:13.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:13 smithi123 ceph-mon[23709]: from='client.? 172.21.15.175:0/44878302' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T10:31:13.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:13 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mon metadata", "id": "smithi175"}]: dispatch 2024-09-21T10:31:13.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:13 smithi123 ceph-mon[23709]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T10:31:13.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:13 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mon metadata", "id": "smithi175"}]: dispatch 2024-09-21T10:31:13.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:13 smithi123 ceph-mon[23709]: mon.smithi175 calling monitor election 2024-09-21T10:31:13.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:13 smithi123 ceph-mon[23709]: from='mgr.? 172.21.15.175:0/2910286930' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi175.txvwrq/crt"}]: dispatch 2024-09-21T10:31:13.352 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:13 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mon metadata", "id": "smithi175"}]: dispatch 2024-09-21T10:31:13.352 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:13 smithi123 ceph-mon[23709]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T10:31:13.352 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:13 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mon metadata", "id": "smithi175"}]: dispatch 2024-09-21T10:31:13.352 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:13 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mon metadata", "id": "smithi175"}]: dispatch 2024-09-21T10:31:13.352 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:13 smithi123 ceph-mon[23709]: mon.smithi123 is new leader, mons smithi123,smithi175 in quorum (ranks 0,1) 2024-09-21T10:31:13.352 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:13 smithi123 ceph-mon[23709]: monmap e2: 2 mons at {smithi123=[v2:172.21.15.123:3300/0,v1:172.21.15.123:6789/0],smithi175=[v2:172.21.15.175:3300/0,v1:172.21.15.175:6789/0]} 2024-09-21T10:31:13.352 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:13 smithi123 ceph-mon[23709]: fsmap 2024-09-21T10:31:13.352 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:13 smithi123 ceph-mon[23709]: osdmap e5: 0 total, 0 up, 0 in 2024-09-21T10:31:13.352 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:13 smithi123 ceph-mon[23709]: mgrmap e17: smithi123.stnuit(active, since 25s) 2024-09-21T10:31:13.352 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:13 smithi123 ceph-mon[23709]: Standby manager daemon smithi175.txvwrq started 2024-09-21T10:31:13.352 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:13 smithi123 ceph-mon[23709]: from='mgr.? 172.21.15.175:0/2910286930' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T10:31:13.352 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:13 smithi123 ceph-mon[23709]: overall HEALTH_OK 2024-09-21T10:31:13.352 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:13 smithi123 ceph-mon[23709]: from='mgr.? 172.21.15.175:0/2910286930' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi175.txvwrq/key"}]: dispatch 2024-09-21T10:31:13.352 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:13 smithi123 ceph-mon[23709]: from='mgr.? 172.21.15.175:0/2910286930' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T10:31:13.353 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:13 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:31:13.598 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-09-21T10:31:13.599 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -- ceph config generate-minimal-conf 2024-09-21T10:31:14.190 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:14 smithi123 ceph-mon[23709]: Deploying daemon node-exporter.smithi175 on smithi175 2024-09-21T10:31:14.190 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:14 smithi123 ceph-mon[23709]: mgrmap e18: smithi123.stnuit(active, since 25s), standbys: smithi175.txvwrq 2024-09-21T10:31:14.190 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:14 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mgr metadata", "who": "smithi175.txvwrq", "id": "smithi175.txvwrq"}]: dispatch 2024-09-21T10:31:14.190 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:14 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mon metadata", "id": "smithi175"}]: dispatch 2024-09-21T10:31:14.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:14 smithi175 ceph-mon[25806]: Deploying daemon node-exporter.smithi175 on smithi175 2024-09-21T10:31:14.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:14 smithi175 ceph-mon[25806]: mgrmap e18: smithi123.stnuit(active, since 25s), standbys: smithi175.txvwrq 2024-09-21T10:31:14.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:14 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mgr metadata", "who": "smithi175.txvwrq", "id": "smithi175.txvwrq"}]: dispatch 2024-09-21T10:31:14.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:14 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mon metadata", "id": "smithi175"}]: dispatch 2024-09-21T10:31:15.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:15 smithi123 ceph-mon[23709]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T10:31:15.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:15 smithi175 ceph-mon[25806]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T10:31:15.413 INFO:teuthology.orchestra.run.smithi123.stdout:# minimal ceph.conf for 3ccd15e6-7804-11ef-baf6-efdc52797490 2024-09-21T10:31:15.413 INFO:teuthology.orchestra.run.smithi123.stdout:[global] 2024-09-21T10:31:15.413 INFO:teuthology.orchestra.run.smithi123.stdout: fsid = 3ccd15e6-7804-11ef-baf6-efdc52797490 2024-09-21T10:31:15.414 INFO:teuthology.orchestra.run.smithi123.stdout: mon_host = [v2:172.21.15.123:3300/0,v1:172.21.15.123:6789/0] [v2:172.21.15.175:3300/0,v1:172.21.15.175:6789/0] 2024-09-21T10:31:15.973 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-09-21T10:31:15.973 DEBUG:teuthology.orchestra.run.smithi123:> set -ex 2024-09-21T10:31:15.973 DEBUG:teuthology.orchestra.run.smithi123:> sudo dd of=/etc/ceph/ceph.conf 2024-09-21T10:31:16.001 DEBUG:teuthology.orchestra.run.smithi123:> set -ex 2024-09-21T10:31:16.001 DEBUG:teuthology.orchestra.run.smithi123:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-21T10:31:16.042 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:16 smithi175 ceph-mon[25806]: from='client.? 172.21.15.123:0/1336443871' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:31:16.068 DEBUG:teuthology.orchestra.run.smithi175:> set -ex 2024-09-21T10:31:16.068 DEBUG:teuthology.orchestra.run.smithi175:> sudo dd of=/etc/ceph/ceph.conf 2024-09-21T10:31:16.098 DEBUG:teuthology.orchestra.run.smithi175:> set -ex 2024-09-21T10:31:16.098 DEBUG:teuthology.orchestra.run.smithi175:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-21T10:31:16.165 DEBUG:tasks.cephadm:set 0 configs 2024-09-21T10:31:16.166 INFO:tasks.cephadm:Deploying OSDs... 2024-09-21T10:31:16.166 DEBUG:teuthology.orchestra.run.smithi123:> set -ex 2024-09-21T10:31:16.166 DEBUG:teuthology.orchestra.run.smithi123:> dd if=/scratch_devs of=/dev/stdout 2024-09-21T10:31:16.182 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-21T10:31:16.182 DEBUG:teuthology.orchestra.run.smithi123:> stat /dev/vg_nvme/lv_1 2024-09-21T10:31:16.237 INFO:teuthology.orchestra.run.smithi123.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-21T10:31:16.237 INFO:teuthology.orchestra.run.smithi123.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T10:31:16.237 INFO:teuthology.orchestra.run.smithi123.stdout:Device: 5h/5d Inode: 665 Links: 1 2024-09-21T10:31:16.237 INFO:teuthology.orchestra.run.smithi123.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T10:31:16.237 INFO:teuthology.orchestra.run.smithi123.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T10:31:16.238 INFO:teuthology.orchestra.run.smithi123.stdout:Access: 2024-09-21 10:30:37.887382831 +0000 2024-09-21T10:31:16.238 INFO:teuthology.orchestra.run.smithi123.stdout:Modify: 2024-09-21 10:28:06.677381844 +0000 2024-09-21T10:31:16.238 INFO:teuthology.orchestra.run.smithi123.stdout:Change: 2024-09-21 10:28:06.677381844 +0000 2024-09-21T10:31:16.238 INFO:teuthology.orchestra.run.smithi123.stdout: Birth: 2024-09-21 10:28:06.677381844 +0000 2024-09-21T10:31:16.238 DEBUG:teuthology.orchestra.run.smithi123:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-21T10:31:16.286 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:16 smithi123 ceph-mon[23709]: from='client.? 172.21.15.123:0/1336443871' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:31:16.301 INFO:teuthology.orchestra.run.smithi123.stderr:1+0 records in 2024-09-21T10:31:16.301 INFO:teuthology.orchestra.run.smithi123.stderr:1+0 records out 2024-09-21T10:31:16.301 INFO:teuthology.orchestra.run.smithi123.stderr:512 bytes copied, 0.000206045 s, 2.5 MB/s 2024-09-21T10:31:16.302 DEBUG:teuthology.orchestra.run.smithi123:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-21T10:31:16.358 DEBUG:teuthology.orchestra.run.smithi123:> stat /dev/vg_nvme/lv_2 2024-09-21T10:31:16.413 INFO:teuthology.orchestra.run.smithi123.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-21T10:31:16.413 INFO:teuthology.orchestra.run.smithi123.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T10:31:16.413 INFO:teuthology.orchestra.run.smithi123.stdout:Device: 5h/5d Inode: 685 Links: 1 2024-09-21T10:31:16.413 INFO:teuthology.orchestra.run.smithi123.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T10:31:16.414 INFO:teuthology.orchestra.run.smithi123.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T10:31:16.414 INFO:teuthology.orchestra.run.smithi123.stdout:Access: 2024-09-21 10:30:37.888382807 +0000 2024-09-21T10:31:16.414 INFO:teuthology.orchestra.run.smithi123.stdout:Modify: 2024-09-21 10:28:06.988374452 +0000 2024-09-21T10:31:16.414 INFO:teuthology.orchestra.run.smithi123.stdout:Change: 2024-09-21 10:28:06.988374452 +0000 2024-09-21T10:31:16.414 INFO:teuthology.orchestra.run.smithi123.stdout: Birth: 2024-09-21 10:28:06.988374452 +0000 2024-09-21T10:31:16.414 DEBUG:teuthology.orchestra.run.smithi123:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-21T10:31:16.477 INFO:teuthology.orchestra.run.smithi123.stderr:1+0 records in 2024-09-21T10:31:16.477 INFO:teuthology.orchestra.run.smithi123.stderr:1+0 records out 2024-09-21T10:31:16.477 INFO:teuthology.orchestra.run.smithi123.stderr:512 bytes copied, 0.000210846 s, 2.4 MB/s 2024-09-21T10:31:16.479 DEBUG:teuthology.orchestra.run.smithi123:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-21T10:31:16.534 DEBUG:teuthology.orchestra.run.smithi123:> stat /dev/vg_nvme/lv_3 2024-09-21T10:31:16.590 INFO:teuthology.orchestra.run.smithi123.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-21T10:31:16.590 INFO:teuthology.orchestra.run.smithi123.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T10:31:16.590 INFO:teuthology.orchestra.run.smithi123.stdout:Device: 5h/5d Inode: 705 Links: 1 2024-09-21T10:31:16.590 INFO:teuthology.orchestra.run.smithi123.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T10:31:16.590 INFO:teuthology.orchestra.run.smithi123.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T10:31:16.590 INFO:teuthology.orchestra.run.smithi123.stdout:Access: 2024-09-21 10:30:37.888382807 +0000 2024-09-21T10:31:16.590 INFO:teuthology.orchestra.run.smithi123.stdout:Modify: 2024-09-21 10:28:07.310366798 +0000 2024-09-21T10:31:16.590 INFO:teuthology.orchestra.run.smithi123.stdout:Change: 2024-09-21 10:28:07.310366798 +0000 2024-09-21T10:31:16.591 INFO:teuthology.orchestra.run.smithi123.stdout: Birth: 2024-09-21 10:28:07.310366798 +0000 2024-09-21T10:31:16.591 DEBUG:teuthology.orchestra.run.smithi123:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-21T10:31:16.654 INFO:teuthology.orchestra.run.smithi123.stderr:1+0 records in 2024-09-21T10:31:16.654 INFO:teuthology.orchestra.run.smithi123.stderr:1+0 records out 2024-09-21T10:31:16.654 INFO:teuthology.orchestra.run.smithi123.stderr:512 bytes copied, 0.000194887 s, 2.6 MB/s 2024-09-21T10:31:16.655 DEBUG:teuthology.orchestra.run.smithi123:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-21T10:31:16.711 DEBUG:teuthology.orchestra.run.smithi123:> stat /dev/vg_nvme/lv_4 2024-09-21T10:31:16.766 INFO:teuthology.orchestra.run.smithi123.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-21T10:31:16.767 INFO:teuthology.orchestra.run.smithi123.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T10:31:16.767 INFO:teuthology.orchestra.run.smithi123.stdout:Device: 5h/5d Inode: 723 Links: 1 2024-09-21T10:31:16.767 INFO:teuthology.orchestra.run.smithi123.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T10:31:16.767 INFO:teuthology.orchestra.run.smithi123.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T10:31:16.767 INFO:teuthology.orchestra.run.smithi123.stdout:Access: 2024-09-21 10:30:37.888382807 +0000 2024-09-21T10:31:16.767 INFO:teuthology.orchestra.run.smithi123.stdout:Modify: 2024-09-21 10:28:07.637359026 +0000 2024-09-21T10:31:16.767 INFO:teuthology.orchestra.run.smithi123.stdout:Change: 2024-09-21 10:28:07.637359026 +0000 2024-09-21T10:31:16.767 INFO:teuthology.orchestra.run.smithi123.stdout: Birth: 2024-09-21 10:28:07.637359026 +0000 2024-09-21T10:31:16.767 DEBUG:teuthology.orchestra.run.smithi123:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-21T10:31:16.831 INFO:teuthology.orchestra.run.smithi123.stderr:1+0 records in 2024-09-21T10:31:16.831 INFO:teuthology.orchestra.run.smithi123.stderr:1+0 records out 2024-09-21T10:31:16.831 INFO:teuthology.orchestra.run.smithi123.stderr:512 bytes copied, 0.00018188 s, 2.8 MB/s 2024-09-21T10:31:16.832 DEBUG:teuthology.orchestra.run.smithi123:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-21T10:31:16.888 DEBUG:teuthology.orchestra.run.smithi175:> set -ex 2024-09-21T10:31:16.888 DEBUG:teuthology.orchestra.run.smithi175:> dd if=/scratch_devs of=/dev/stdout 2024-09-21T10:31:16.904 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-21T10:31:16.904 DEBUG:teuthology.orchestra.run.smithi175:> stat /dev/vg_nvme/lv_1 2024-09-21T10:31:16.959 INFO:teuthology.orchestra.run.smithi175.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-21T10:31:16.959 INFO:teuthology.orchestra.run.smithi175.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T10:31:16.959 INFO:teuthology.orchestra.run.smithi175.stdout:Device: 5h/5d Inode: 666 Links: 1 2024-09-21T10:31:16.959 INFO:teuthology.orchestra.run.smithi175.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T10:31:16.959 INFO:teuthology.orchestra.run.smithi175.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T10:31:16.959 INFO:teuthology.orchestra.run.smithi175.stdout:Access: 2024-09-21 10:31:13.914489208 +0000 2024-09-21T10:31:16.959 INFO:teuthology.orchestra.run.smithi175.stdout:Modify: 2024-09-21 10:28:06.658379119 +0000 2024-09-21T10:31:16.959 INFO:teuthology.orchestra.run.smithi175.stdout:Change: 2024-09-21 10:28:06.658379119 +0000 2024-09-21T10:31:16.959 INFO:teuthology.orchestra.run.smithi175.stdout: Birth: 2024-09-21 10:28:06.658379119 +0000 2024-09-21T10:31:16.960 DEBUG:teuthology.orchestra.run.smithi175:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-21T10:31:17.023 INFO:teuthology.orchestra.run.smithi175.stderr:1+0 records in 2024-09-21T10:31:17.023 INFO:teuthology.orchestra.run.smithi175.stderr:1+0 records out 2024-09-21T10:31:17.023 INFO:teuthology.orchestra.run.smithi175.stderr:512 bytes copied, 0.000196562 s, 2.6 MB/s 2024-09-21T10:31:17.024 DEBUG:teuthology.orchestra.run.smithi175:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-21T10:31:17.079 DEBUG:teuthology.orchestra.run.smithi175:> stat /dev/vg_nvme/lv_2 2024-09-21T10:31:17.135 INFO:teuthology.orchestra.run.smithi175.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-21T10:31:17.135 INFO:teuthology.orchestra.run.smithi175.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T10:31:17.135 INFO:teuthology.orchestra.run.smithi175.stdout:Device: 5h/5d Inode: 687 Links: 1 2024-09-21T10:31:17.135 INFO:teuthology.orchestra.run.smithi175.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T10:31:17.135 INFO:teuthology.orchestra.run.smithi175.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T10:31:17.135 INFO:teuthology.orchestra.run.smithi175.stdout:Access: 2024-09-21 10:31:13.914489208 +0000 2024-09-21T10:31:17.136 INFO:teuthology.orchestra.run.smithi175.stdout:Modify: 2024-09-21 10:28:06.993374293 +0000 2024-09-21T10:31:17.136 INFO:teuthology.orchestra.run.smithi175.stdout:Change: 2024-09-21 10:28:06.993374293 +0000 2024-09-21T10:31:17.136 INFO:teuthology.orchestra.run.smithi175.stdout: Birth: 2024-09-21 10:28:06.993374293 +0000 2024-09-21T10:31:17.136 DEBUG:teuthology.orchestra.run.smithi175:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-21T10:31:17.197 INFO:teuthology.orchestra.run.smithi175.stderr:1+0 records in 2024-09-21T10:31:17.197 INFO:teuthology.orchestra.run.smithi175.stderr:1+0 records out 2024-09-21T10:31:17.198 INFO:teuthology.orchestra.run.smithi175.stderr:512 bytes copied, 0.000216303 s, 2.4 MB/s 2024-09-21T10:31:17.199 DEBUG:teuthology.orchestra.run.smithi175:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-21T10:31:17.255 DEBUG:teuthology.orchestra.run.smithi175:> stat /dev/vg_nvme/lv_3 2024-09-21T10:31:17.310 INFO:teuthology.orchestra.run.smithi175.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-21T10:31:17.310 INFO:teuthology.orchestra.run.smithi175.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T10:31:17.310 INFO:teuthology.orchestra.run.smithi175.stdout:Device: 5h/5d Inode: 703 Links: 1 2024-09-21T10:31:17.310 INFO:teuthology.orchestra.run.smithi175.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T10:31:17.310 INFO:teuthology.orchestra.run.smithi175.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T10:31:17.310 INFO:teuthology.orchestra.run.smithi175.stdout:Access: 2024-09-21 10:31:13.915489193 +0000 2024-09-21T10:31:17.310 INFO:teuthology.orchestra.run.smithi175.stdout:Modify: 2024-09-21 10:28:07.307369768 +0000 2024-09-21T10:31:17.310 INFO:teuthology.orchestra.run.smithi175.stdout:Change: 2024-09-21 10:28:07.307369768 +0000 2024-09-21T10:31:17.310 INFO:teuthology.orchestra.run.smithi175.stdout: Birth: 2024-09-21 10:28:07.307369768 +0000 2024-09-21T10:31:17.311 DEBUG:teuthology.orchestra.run.smithi175:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-21T10:31:17.350 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:17 smithi123 ceph-mon[23709]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T10:31:17.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:17 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:31:17.370 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:17 smithi175 ceph-mon[25806]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T10:31:17.370 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:17 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:31:17.373 INFO:teuthology.orchestra.run.smithi175.stderr:1+0 records in 2024-09-21T10:31:17.373 INFO:teuthology.orchestra.run.smithi175.stderr:1+0 records out 2024-09-21T10:31:17.373 INFO:teuthology.orchestra.run.smithi175.stderr:512 bytes copied, 0.000187541 s, 2.7 MB/s 2024-09-21T10:31:17.374 DEBUG:teuthology.orchestra.run.smithi175:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-21T10:31:17.430 DEBUG:teuthology.orchestra.run.smithi175:> stat /dev/vg_nvme/lv_4 2024-09-21T10:31:17.485 INFO:teuthology.orchestra.run.smithi175.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-21T10:31:17.485 INFO:teuthology.orchestra.run.smithi175.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T10:31:17.485 INFO:teuthology.orchestra.run.smithi175.stdout:Device: 5h/5d Inode: 724 Links: 1 2024-09-21T10:31:17.485 INFO:teuthology.orchestra.run.smithi175.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T10:31:17.485 INFO:teuthology.orchestra.run.smithi175.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T10:31:17.485 INFO:teuthology.orchestra.run.smithi175.stdout:Access: 2024-09-21 10:31:13.915489193 +0000 2024-09-21T10:31:17.485 INFO:teuthology.orchestra.run.smithi175.stdout:Modify: 2024-09-21 10:28:07.639364985 +0000 2024-09-21T10:31:17.485 INFO:teuthology.orchestra.run.smithi175.stdout:Change: 2024-09-21 10:28:07.639364985 +0000 2024-09-21T10:31:17.485 INFO:teuthology.orchestra.run.smithi175.stdout: Birth: 2024-09-21 10:28:07.639364985 +0000 2024-09-21T10:31:17.485 DEBUG:teuthology.orchestra.run.smithi175:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-21T10:31:17.548 INFO:teuthology.orchestra.run.smithi175.stderr:1+0 records in 2024-09-21T10:31:17.548 INFO:teuthology.orchestra.run.smithi175.stderr:1+0 records out 2024-09-21T10:31:17.548 INFO:teuthology.orchestra.run.smithi175.stderr:512 bytes copied, 0.000290257 s, 1.8 MB/s 2024-09-21T10:31:17.549 DEBUG:teuthology.orchestra.run.smithi175:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-21T10:31:17.605 INFO:tasks.cephadm:Deploying osd.0 on smithi123 with /dev/vg_nvme/lv_4... 2024-09-21T10:31:17.605 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -- lvm zap /dev/vg_nvme/lv_4 2024-09-21T10:31:18.160 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:18 smithi123 ceph-mon[23709]: Deploying daemon prometheus.smithi123 on smithi123 2024-09-21T10:31:18.160 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:18 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:31:18.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:18 smithi175 ceph-mon[25806]: Deploying daemon prometheus.smithi123 on smithi123 2024-09-21T10:31:18.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:18 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:31:19.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:19 smithi123 ceph-mon[23709]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T10:31:19.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:19 smithi175 ceph-mon[25806]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T10:31:21.350 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:21 smithi123 ceph-mon[23709]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T10:31:21.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:21 smithi175 ceph-mon[25806]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T10:31:23.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:23 smithi123 ceph-mon[23709]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T10:31:23.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:23 smithi175 ceph-mon[25806]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T10:31:25.350 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:25 smithi123 ceph-mon[23709]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T10:31:25.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:25 smithi175 ceph-mon[25806]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T10:31:26.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:26 smithi123 ceph-mon[23709]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T10:31:26.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:26 smithi175 ceph-mon[25806]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T10:31:28.850 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:28 smithi123 ceph-mon[23709]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T10:31:29.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:28 smithi175 ceph-mon[25806]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T10:31:30.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:30 smithi175 ceph-mon[25806]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T10:31:30.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:30 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:31:30.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:30 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:31:30.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:30 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:31:30.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:30 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T10:31:30.930 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:30 smithi123 ceph-mon[23709]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T10:31:30.930 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:30 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:31:30.930 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:30 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:31:30.930 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:30 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:31:30.930 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:30 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T10:31:32.099 INFO:teuthology.orchestra.run.smithi123.stdout: 2024-09-21T10:31:32.117 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -- ceph orch daemon add osd smithi123:vg_nvme/lv_4 2024-09-21T10:31:32.732 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:32 smithi123 ceph-mon[23709]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T10:31:32.732 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:32 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:31:32.732 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:32 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:31:32.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:32 smithi175 ceph-mon[25806]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T10:31:32.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:32 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:31:32.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:32 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:31:33.738 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:33 smithi123 ceph-mon[23709]: Updating smithi175:/etc/ceph/ceph.conf 2024-09-21T10:31:33.738 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:33 smithi123 ceph-mon[23709]: Updating smithi175:/etc/ceph/ceph.client.admin.keyring 2024-09-21T10:31:33.738 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:33 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:31:34.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:33 smithi175 ceph-mon[25806]: Updating smithi175:/etc/ceph/ceph.conf 2024-09-21T10:31:34.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:33 smithi175 ceph-mon[25806]: Updating smithi175:/etc/ceph/ceph.client.admin.keyring 2024-09-21T10:31:34.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:33 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:31:34.996 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:34 smithi123 ceph-mon[23709]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T10:31:34.996 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:34 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T10:31:34.996 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:34 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T10:31:34.996 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:34 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:31:35.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:34 smithi175 ceph-mon[25806]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T10:31:35.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:34 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T10:31:35.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:34 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T10:31:35.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:34 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:31:35.843 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:35 smithi123 ceph-mon[23709]: from='client.14232 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi123:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:31:36.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:35 smithi175 ceph-mon[25806]: from='client.14232 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi123:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:31:36.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:36 smithi123 ceph-mon[23709]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T10:31:37.024 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:36 smithi175 ceph-mon[25806]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T10:31:37.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:37 smithi123 ceph-mon[23709]: from='client.? 172.21.15.123:0/2425270158' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a0c9cae5-8e1a-453f-b682-0b656123004e"}]: dispatch 2024-09-21T10:31:37.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:37 smithi123 ceph-mon[23709]: from='client.? 172.21.15.123:0/2425270158' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a0c9cae5-8e1a-453f-b682-0b656123004e"}]': finished 2024-09-21T10:31:37.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:37 smithi123 ceph-mon[23709]: osdmap e6: 1 total, 0 up, 1 in 2024-09-21T10:31:37.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:37 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T10:31:37.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:37 smithi123 ceph-mon[23709]: from='client.? 172.21.15.123:0/2911821934' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T10:31:37.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:37 smithi175 ceph-mon[25806]: from='client.? 172.21.15.123:0/2425270158' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a0c9cae5-8e1a-453f-b682-0b656123004e"}]: dispatch 2024-09-21T10:31:37.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:37 smithi175 ceph-mon[25806]: from='client.? 172.21.15.123:0/2425270158' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a0c9cae5-8e1a-453f-b682-0b656123004e"}]': finished 2024-09-21T10:31:37.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:37 smithi175 ceph-mon[25806]: osdmap e6: 1 total, 0 up, 1 in 2024-09-21T10:31:37.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:37 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T10:31:37.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:37 smithi175 ceph-mon[25806]: from='client.? 172.21.15.123:0/2911821934' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T10:31:39.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:38 smithi123 ceph-mon[23709]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T10:31:39.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:38 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:31:39.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:38 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:31:39.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:38 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:31:39.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:38 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T10:31:39.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:38 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T10:31:39.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:38 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:31:39.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:38 smithi175 ceph-mon[25806]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T10:31:39.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:38 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:31:39.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:38 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:31:39.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:38 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:31:39.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:38 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T10:31:39.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:38 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T10:31:39.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:38 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:31:40.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:39 smithi123 ceph-mon[23709]: Updating smithi123:/etc/ceph/ceph.conf 2024-09-21T10:31:40.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:39 smithi123 ceph-mon[23709]: Reconfiguring mon.smithi123 (unknown last config time)... 2024-09-21T10:31:40.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:39 smithi123 ceph-mon[23709]: Reconfiguring daemon mon.smithi123 on smithi123 2024-09-21T10:31:40.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:39 smithi175 ceph-mon[25806]: Updating smithi123:/etc/ceph/ceph.conf 2024-09-21T10:31:40.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:39 smithi175 ceph-mon[25806]: Reconfiguring mon.smithi123 (unknown last config time)... 2024-09-21T10:31:40.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:39 smithi175 ceph-mon[25806]: Reconfiguring daemon mon.smithi123 on smithi123 2024-09-21T10:31:41.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:40 smithi123 ceph-mon[23709]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T10:31:41.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:40 smithi175 ceph-mon[25806]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T10:31:43.196 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:42 smithi123 ceph-mon[23709]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T10:31:43.196 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:42 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:31:43.196 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:42 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi123.stnuit", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T10:31:43.196 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:42 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T10:31:43.197 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:42 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:31:43.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:42 smithi175 ceph-mon[25806]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T10:31:43.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:42 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:31:43.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:42 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi123.stnuit", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T10:31:43.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:42 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T10:31:43.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:42 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:31:44.215 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:43 smithi123 ceph-mon[23709]: Reconfiguring mgr.smithi123.stnuit (unknown last config time)... 2024-09-21T10:31:44.215 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:43 smithi123 ceph-mon[23709]: Reconfiguring daemon mgr.smithi123.stnuit on smithi123 2024-09-21T10:31:44.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:43 smithi175 ceph-mon[25806]: Reconfiguring mgr.smithi123.stnuit (unknown last config time)... 2024-09-21T10:31:44.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:43 smithi175 ceph-mon[25806]: Reconfiguring daemon mgr.smithi123.stnuit on smithi123 2024-09-21T10:31:45.075 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:44 smithi123 ceph-mon[23709]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T10:31:45.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:44 smithi175 ceph-mon[25806]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T10:31:46.513 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:46 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:31:46.513 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:46 smithi123 ceph-mon[23709]: Reconfiguring alertmanager.smithi123 (dependencies changed)... 2024-09-21T10:31:46.513 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:46 smithi123 ceph-mon[23709]: Reconfiguring daemon alertmanager.smithi123 on smithi123 2024-09-21T10:31:46.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:46 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:31:46.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:46 smithi175 ceph-mon[25806]: Reconfiguring alertmanager.smithi123 (dependencies changed)... 2024-09-21T10:31:46.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:46 smithi175 ceph-mon[25806]: Reconfiguring daemon alertmanager.smithi123 on smithi123 2024-09-21T10:31:47.553 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:47 smithi123 ceph-mon[23709]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T10:31:47.554 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:47 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-21T10:31:47.554 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:47 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:31:47.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:47 smithi175 ceph-mon[25806]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T10:31:47.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:47 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-21T10:31:47.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:47 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:31:48.415 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:48 smithi123 ceph-mon[23709]: Deploying daemon osd.0 on smithi123 2024-09-21T10:31:48.415 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:48 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.stnuit/mirror_snapshot_schedule"}]: dispatch 2024-09-21T10:31:48.416 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:48 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.stnuit/trash_purge_schedule"}]: dispatch 2024-09-21T10:31:48.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:48 smithi175 ceph-mon[25806]: Deploying daemon osd.0 on smithi123 2024-09-21T10:31:48.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:48 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.stnuit/mirror_snapshot_schedule"}]: dispatch 2024-09-21T10:31:48.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:48 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.stnuit/trash_purge_schedule"}]: dispatch 2024-09-21T10:31:49.320 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:49 smithi123 ceph-mon[23709]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T10:31:49.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:49 smithi175 ceph-mon[25806]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T10:31:51.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:51 smithi123 ceph-mon[23709]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T10:31:51.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:51 smithi175 ceph-mon[25806]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T10:31:52.403 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:52 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:31:52.403 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:52 smithi123 ceph-mon[23709]: Reconfiguring crash.smithi123 (monmap changed)... 2024-09-21T10:31:52.403 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:52 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi123", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-21T10:31:52.403 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:52 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:31:52.403 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:52 smithi123 ceph-mon[23709]: Reconfiguring daemon crash.smithi123 on smithi123 2024-09-21T10:31:52.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:52 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:31:52.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:52 smithi175 ceph-mon[25806]: Reconfiguring crash.smithi123 (monmap changed)... 2024-09-21T10:31:52.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:52 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi123", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-21T10:31:52.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:52 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:31:52.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:52 smithi175 ceph-mon[25806]: Reconfiguring daemon crash.smithi123 on smithi123 2024-09-21T10:31:53.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:53 smithi123 ceph-mon[23709]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T10:31:53.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:53 smithi175 ceph-mon[25806]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T10:31:55.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:55 smithi123 ceph-mon[23709]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T10:31:55.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:55 smithi175 ceph-mon[25806]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T10:31:56.733 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:56 smithi123 ceph-mon[23709]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T10:31:56.733 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:56 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:31:56.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:56 smithi175 ceph-mon[25806]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T10:31:56.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:56 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:31:58.767 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:58 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:31:58.767 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:58 smithi123 ceph-mon[23709]: Reconfiguring grafana.smithi123 (dependencies changed)... 2024-09-21T10:31:58.767 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:58 smithi123 ceph-mon[23709]: Reconfiguring daemon grafana.smithi123 on smithi123 2024-09-21T10:31:58.767 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:58 smithi123 ceph-mon[23709]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T10:31:58.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:58 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:31:58.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:58 smithi175 ceph-mon[25806]: Reconfiguring grafana.smithi123 (dependencies changed)... 2024-09-21T10:31:58.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:58 smithi175 ceph-mon[25806]: Reconfiguring daemon grafana.smithi123 on smithi123 2024-09-21T10:31:58.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:58 smithi175 ceph-mon[25806]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T10:31:59.822 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:31:59 smithi175 ceph-mon[25806]: from='osd.0 [v2:172.21.15.123:6802/1645528985,v1:172.21.15.123:6803/1645528985]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-21T10:31:59.850 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:31:59 smithi123 ceph-mon[23709]: from='osd.0 [v2:172.21.15.123:6802/1645528985,v1:172.21.15.123:6803/1645528985]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-21T10:32:00.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:00 smithi123 ceph-mon[23709]: from='osd.0 [v2:172.21.15.123:6802/1645528985,v1:172.21.15.123:6803/1645528985]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-21T10:32:00.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:00 smithi123 ceph-mon[23709]: osdmap e7: 1 total, 0 up, 1 in 2024-09-21T10:32:00.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:00 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T10:32:00.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:00 smithi123 ceph-mon[23709]: from='osd.0 [v2:172.21.15.123:6802/1645528985,v1:172.21.15.123:6803/1645528985]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]: dispatch 2024-09-21T10:32:00.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:00 smithi123 ceph-mon[23709]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T10:32:00.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:00 smithi175 ceph-mon[25806]: from='osd.0 [v2:172.21.15.123:6802/1645528985,v1:172.21.15.123:6803/1645528985]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-21T10:32:00.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:00 smithi175 ceph-mon[25806]: osdmap e7: 1 total, 0 up, 1 in 2024-09-21T10:32:00.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:00 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T10:32:00.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:00 smithi175 ceph-mon[25806]: from='osd.0 [v2:172.21.15.123:6802/1645528985,v1:172.21.15.123:6803/1645528985]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]: dispatch 2024-09-21T10:32:00.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:00 smithi175 ceph-mon[25806]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T10:32:01.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:01 smithi123 ceph-mon[23709]: from='osd.0 [v2:172.21.15.123:6802/1645528985,v1:172.21.15.123:6803/1645528985]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]': finished 2024-09-21T10:32:01.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:01 smithi123 ceph-mon[23709]: osdmap e8: 1 total, 0 up, 1 in 2024-09-21T10:32:01.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:01 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T10:32:01.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:01 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T10:32:01.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:01 smithi175 ceph-mon[25806]: from='osd.0 [v2:172.21.15.123:6802/1645528985,v1:172.21.15.123:6803/1645528985]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]': finished 2024-09-21T10:32:01.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:01 smithi175 ceph-mon[25806]: osdmap e8: 1 total, 0 up, 1 in 2024-09-21T10:32:01.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:01 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T10:32:01.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:01 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T10:32:02.743 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:02 smithi123 ceph-mon[23709]: purged_snaps scrub starts 2024-09-21T10:32:02.743 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:02 smithi123 ceph-mon[23709]: purged_snaps scrub ok 2024-09-21T10:32:02.743 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:02 smithi123 ceph-mon[23709]: osd.0 [v2:172.21.15.123:6802/1645528985,v1:172.21.15.123:6803/1645528985] boot 2024-09-21T10:32:02.743 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:02 smithi123 ceph-mon[23709]: osdmap e9: 1 total, 1 up, 1 in 2024-09-21T10:32:02.743 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:02 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T10:32:02.743 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:02 smithi123 ceph-mon[23709]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T10:32:02.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:02 smithi175 ceph-mon[25806]: purged_snaps scrub starts 2024-09-21T10:32:02.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:02 smithi175 ceph-mon[25806]: purged_snaps scrub ok 2024-09-21T10:32:02.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:02 smithi175 ceph-mon[25806]: osd.0 [v2:172.21.15.123:6802/1645528985,v1:172.21.15.123:6803/1645528985] boot 2024-09-21T10:32:02.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:02 smithi175 ceph-mon[25806]: osdmap e9: 1 total, 1 up, 1 in 2024-09-21T10:32:02.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:02 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T10:32:02.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:02 smithi175 ceph-mon[25806]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T10:32:03.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:03 smithi175 ceph-mon[25806]: osdmap e10: 1 total, 1 up, 1 in 2024-09-21T10:32:03.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:03 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:32:03.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:03 smithi175 ceph-mon[25806]: Reconfiguring crash.smithi175 (monmap changed)... 2024-09-21T10:32:03.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:03 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi175", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-21T10:32:03.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:03 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:32:03.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:03 smithi175 ceph-mon[25806]: Reconfiguring daemon crash.smithi175 on smithi175 2024-09-21T10:32:03.845 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:03 smithi123 ceph-mon[23709]: osdmap e10: 1 total, 1 up, 1 in 2024-09-21T10:32:03.845 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:03 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:32:03.846 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:03 smithi123 ceph-mon[23709]: Reconfiguring crash.smithi175 (monmap changed)... 2024-09-21T10:32:03.846 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:03 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi175", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-21T10:32:03.846 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:03 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:32:03.846 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:03 smithi123 ceph-mon[23709]: Reconfiguring daemon crash.smithi175 on smithi175 2024-09-21T10:32:04.201 INFO:teuthology.orchestra.run.smithi123.stdout:Created osd(s) 0 on host 'smithi123' 2024-09-21T10:32:04.805 DEBUG:teuthology.orchestra.run.smithi123:osd.0> sudo journalctl -f -n 0 -u ceph-3ccd15e6-7804-11ef-baf6-efdc52797490@osd.0.service 2024-09-21T10:32:04.808 INFO:tasks.cephadm:Deploying osd.1 on smithi123 with /dev/vg_nvme/lv_3... 2024-09-21T10:32:04.808 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -- lvm zap /dev/vg_nvme/lv_3 2024-09-21T10:32:05.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:04 smithi123 ceph-mon[23709]: pgmap v36: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-21T10:32:05.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:04 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:32:05.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:04 smithi123 ceph-mon[23709]: Reconfiguring mgr.smithi175.txvwrq (monmap changed)... 2024-09-21T10:32:05.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:04 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi175.txvwrq", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T10:32:05.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:04 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T10:32:05.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:04 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:32:05.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:04 smithi123 ceph-mon[23709]: Reconfiguring daemon mgr.smithi175.txvwrq on smithi175 2024-09-21T10:32:05.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:04 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:32:05.255 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:04 smithi175 ceph-mon[25806]: pgmap v36: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-21T10:32:05.255 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:04 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:32:05.255 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:04 smithi175 ceph-mon[25806]: Reconfiguring mgr.smithi175.txvwrq (monmap changed)... 2024-09-21T10:32:05.255 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:04 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi175.txvwrq", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T10:32:05.255 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:04 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T10:32:05.255 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:04 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:32:05.255 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:04 smithi175 ceph-mon[25806]: Reconfiguring daemon mgr.smithi175.txvwrq on smithi175 2024-09-21T10:32:05.255 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:04 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:32:06.329 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:06 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:32:06.329 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:06 smithi123 ceph-mon[23709]: Reconfiguring mon.smithi175 (monmap changed)... 2024-09-21T10:32:06.329 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:06 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T10:32:06.330 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:06 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T10:32:06.330 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:06 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:32:06.330 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:06 smithi123 ceph-mon[23709]: Reconfiguring daemon mon.smithi175 on smithi175 2024-09-21T10:32:06.330 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:06 smithi123 ceph-mon[23709]: pgmap v37: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-21T10:32:06.622 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:06 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:32:06.622 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:06 smithi175 ceph-mon[25806]: Reconfiguring mon.smithi175 (monmap changed)... 2024-09-21T10:32:06.622 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:06 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T10:32:06.622 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:06 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T10:32:06.622 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:06 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:32:06.622 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:06 smithi175 ceph-mon[25806]: Reconfiguring daemon mon.smithi175 on smithi175 2024-09-21T10:32:06.622 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:06 smithi175 ceph-mon[25806]: pgmap v37: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-21T10:32:07.425 INFO:teuthology.orchestra.run.smithi123.stdout: 2024-09-21T10:32:07.443 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -- ceph orch daemon add osd smithi123:vg_nvme/lv_3 2024-09-21T10:32:07.731 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:07 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:32:07.731 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:07 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T10:32:07.732 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:07 smithi123 ceph-mon[23709]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T10:32:07.732 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:07 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.123:9093"}]: dispatch 2024-09-21T10:32:07.732 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:07 smithi123 ceph-mon[23709]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.123:9093"}]: dispatch 2024-09-21T10:32:07.732 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:07 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:32:07.732 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:07 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T10:32:07.732 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:07 smithi123 ceph-mon[23709]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T10:32:07.732 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:07 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.123:3000"}]: dispatch 2024-09-21T10:32:07.732 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:07 smithi123 ceph-mon[23709]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.123:3000"}]: dispatch 2024-09-21T10:32:07.732 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:07 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:32:07.732 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:07 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T10:32:07.732 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:07 smithi123 ceph-mon[23709]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T10:32:07.732 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:07 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.123:9095"}]: dispatch 2024-09-21T10:32:07.732 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:07 smithi123 ceph-mon[23709]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.123:9095"}]: dispatch 2024-09-21T10:32:07.732 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:07 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:32:07.733 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:07 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:32:07.733 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:07 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:32:07.733 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:07 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T10:32:07.764 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:07 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:32:07.765 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:07 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T10:32:07.765 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:07 smithi175 ceph-mon[25806]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T10:32:07.765 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:07 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.123:9093"}]: dispatch 2024-09-21T10:32:07.765 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:07 smithi175 ceph-mon[25806]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.123:9093"}]: dispatch 2024-09-21T10:32:07.765 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:07 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:32:07.765 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:07 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T10:32:07.765 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:07 smithi175 ceph-mon[25806]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T10:32:07.765 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:07 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.123:3000"}]: dispatch 2024-09-21T10:32:07.765 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:07 smithi175 ceph-mon[25806]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.123:3000"}]: dispatch 2024-09-21T10:32:07.765 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:07 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:32:07.765 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:07 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T10:32:07.765 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:07 smithi175 ceph-mon[25806]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T10:32:07.765 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:07 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.123:9095"}]: dispatch 2024-09-21T10:32:07.766 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:07 smithi175 ceph-mon[25806]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.123:9095"}]: dispatch 2024-09-21T10:32:07.766 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:07 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:32:07.766 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:07 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:32:07.766 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:07 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:32:07.766 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:07 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T10:32:09.421 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:09 smithi123 ceph-mon[23709]: pgmap v38: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-21T10:32:09.422 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:09 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:32:09.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:09 smithi175 ceph-mon[25806]: pgmap v38: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-21T10:32:09.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:09 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:32:11.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:11 smithi123 ceph-mon[23709]: pgmap v39: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-21T10:32:11.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:11 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T10:32:11.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:11 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T10:32:11.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:11 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:32:11.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:11 smithi175 ceph-mon[25806]: pgmap v39: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-21T10:32:11.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:11 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T10:32:11.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:11 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T10:32:11.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:11 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:32:12.430 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:12 smithi123 ceph-mon[23709]: from='client.14250 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi123:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:32:12.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:12 smithi175 ceph-mon[25806]: from='client.14250 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi123:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:32:13.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:13 smithi123 ceph-mon[23709]: pgmap v40: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-21T10:32:13.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:13 smithi123 ceph-mon[23709]: from='client.? 172.21.15.123:0/1954693053' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f686f59d-6dbf-47c4-8ff0-3f2dbf64e1e1"}]: dispatch 2024-09-21T10:32:13.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:13 smithi175 ceph-mon[25806]: pgmap v40: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-21T10:32:13.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:13 smithi175 ceph-mon[25806]: from='client.? 172.21.15.123:0/1954693053' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f686f59d-6dbf-47c4-8ff0-3f2dbf64e1e1"}]: dispatch 2024-09-21T10:32:14.516 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:14 smithi123 ceph-mon[23709]: from='client.? 172.21.15.123:0/1954693053' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f686f59d-6dbf-47c4-8ff0-3f2dbf64e1e1"}]': finished 2024-09-21T10:32:14.516 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:14 smithi123 ceph-mon[23709]: osdmap e11: 2 total, 1 up, 2 in 2024-09-21T10:32:14.516 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:14 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T10:32:14.516 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:14 smithi123 ceph-mon[23709]: from='client.? 172.21.15.123:0/1416472190' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T10:32:14.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:14 smithi175 ceph-mon[25806]: from='client.? 172.21.15.123:0/1954693053' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f686f59d-6dbf-47c4-8ff0-3f2dbf64e1e1"}]': finished 2024-09-21T10:32:14.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:14 smithi175 ceph-mon[25806]: osdmap e11: 2 total, 1 up, 2 in 2024-09-21T10:32:14.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:14 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T10:32:14.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:14 smithi175 ceph-mon[25806]: from='client.? 172.21.15.123:0/1416472190' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T10:32:15.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:15 smithi123 ceph-mon[23709]: pgmap v42: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-21T10:32:15.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:15 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:32:15.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:15 smithi175 ceph-mon[25806]: pgmap v42: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-21T10:32:15.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:15 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:32:16.744 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:16 smithi123 ceph-mon[23709]: pgmap v43: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-21T10:32:16.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:16 smithi175 ceph-mon[25806]: pgmap v43: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-21T10:32:18.772 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:18 smithi123 ceph-mon[23709]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-21T10:32:18.869 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:18 smithi175 ceph-mon[25806]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-21T10:32:20.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:20 smithi123 ceph-mon[23709]: Detected new or changed devices on smithi123 2024-09-21T10:32:20.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:20 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:32:20.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:20 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T10:32:20.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:20 smithi123 ceph-mon[23709]: Adjusting osd_memory_target on smithi123 to 12864M 2024-09-21T10:32:20.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:20 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:32:20.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:20 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:32:20.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:20 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:32:20.352 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:20 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:32:20.352 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:20 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T10:32:20.352 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:20 smithi123 ceph-mon[23709]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-21T10:32:20.596 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:20 smithi175 ceph-mon[25806]: Detected new or changed devices on smithi123 2024-09-21T10:32:20.596 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:20 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:32:20.596 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:20 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T10:32:20.596 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:20 smithi175 ceph-mon[25806]: Adjusting osd_memory_target on smithi123 to 12864M 2024-09-21T10:32:20.596 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:20 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:32:20.596 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:20 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:32:20.597 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:20 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:32:20.597 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:20 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:32:20.597 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:20 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T10:32:20.597 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:20 smithi175 ceph-mon[25806]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-21T10:32:21.850 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:21 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-21T10:32:21.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:21 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:32:21.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:21 smithi123 ceph-mon[23709]: Deploying daemon osd.1 on smithi123 2024-09-21T10:32:21.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:21 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:32:21.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:21 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-21T10:32:21.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:21 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:32:21.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:21 smithi175 ceph-mon[25806]: Deploying daemon osd.1 on smithi123 2024-09-21T10:32:21.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:21 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:32:22.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:22 smithi123 ceph-mon[23709]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-21T10:32:22.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:22 smithi175 ceph-mon[25806]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-21T10:32:24.829 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:24 smithi123 ceph-mon[23709]: pgmap v47: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-21T10:32:24.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:24 smithi175 ceph-mon[25806]: pgmap v47: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-21T10:32:27.019 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:26 smithi123 ceph-mon[23709]: pgmap v48: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-21T10:32:27.019 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:26 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:32:27.019 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:26 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:32:27.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:26 smithi175 ceph-mon[25806]: pgmap v48: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-21T10:32:27.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:26 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:32:27.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:26 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:32:28.922 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:28 smithi123 ceph-mon[23709]: pgmap v49: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-21T10:32:29.017 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:28 smithi175 ceph-mon[25806]: pgmap v49: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-21T10:32:30.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:29 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:32:30.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:29 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:32:30.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:29 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:32:30.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:29 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T10:32:30.328 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:29 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:32:30.328 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:29 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:32:30.328 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:29 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:32:30.328 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:29 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T10:32:31.238 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:30 smithi123 ceph-mon[23709]: pgmap v50: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-21T10:32:31.238 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:30 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:32:31.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:30 smithi175 ceph-mon[25806]: pgmap v50: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-21T10:32:31.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:30 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:32:32.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:31 smithi123 ceph-mon[23709]: from='osd.1 [v2:172.21.15.123:6810/774521400,v1:172.21.15.123:6811/774521400]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-21T10:32:32.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:31 smithi175 ceph-mon[25806]: from='osd.1 [v2:172.21.15.123:6810/774521400,v1:172.21.15.123:6811/774521400]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-21T10:32:32.863 INFO:teuthology.orchestra.run.smithi123.stdout:Created osd(s) 1 on host 'smithi123' 2024-09-21T10:32:33.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:32 smithi123 ceph-mon[23709]: pgmap v51: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-21T10:32:33.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:32 smithi123 ceph-mon[23709]: from='osd.1 [v2:172.21.15.123:6810/774521400,v1:172.21.15.123:6811/774521400]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-21T10:32:33.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:32 smithi123 ceph-mon[23709]: osdmap e12: 2 total, 1 up, 2 in 2024-09-21T10:32:33.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:32 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T10:32:33.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:32 smithi123 ceph-mon[23709]: from='osd.1 [v2:172.21.15.123:6810/774521400,v1:172.21.15.123:6811/774521400]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]: dispatch 2024-09-21T10:32:33.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:32 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:32:33.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:32 smithi175 ceph-mon[25806]: pgmap v51: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-21T10:32:33.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:32 smithi175 ceph-mon[25806]: from='osd.1 [v2:172.21.15.123:6810/774521400,v1:172.21.15.123:6811/774521400]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-21T10:32:33.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:32 smithi175 ceph-mon[25806]: osdmap e12: 2 total, 1 up, 2 in 2024-09-21T10:32:33.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:32 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T10:32:33.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:32 smithi175 ceph-mon[25806]: from='osd.1 [v2:172.21.15.123:6810/774521400,v1:172.21.15.123:6811/774521400]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]: dispatch 2024-09-21T10:32:33.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:32 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:32:34.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:33 smithi123 ceph-mon[23709]: from='osd.1 [v2:172.21.15.123:6810/774521400,v1:172.21.15.123:6811/774521400]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]': finished 2024-09-21T10:32:34.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:33 smithi123 ceph-mon[23709]: osdmap e13: 2 total, 1 up, 2 in 2024-09-21T10:32:34.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:33 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T10:32:34.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:33 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T10:32:34.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:33 smithi175 ceph-mon[25806]: from='osd.1 [v2:172.21.15.123:6810/774521400,v1:172.21.15.123:6811/774521400]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]': finished 2024-09-21T10:32:34.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:33 smithi175 ceph-mon[25806]: osdmap e13: 2 total, 1 up, 2 in 2024-09-21T10:32:34.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:33 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T10:32:34.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:33 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T10:32:34.842 DEBUG:teuthology.orchestra.run.smithi123:osd.1> sudo journalctl -f -n 0 -u ceph-3ccd15e6-7804-11ef-baf6-efdc52797490@osd.1.service 2024-09-21T10:32:34.844 INFO:tasks.cephadm:Deploying osd.2 on smithi123 with /dev/vg_nvme/lv_2... 2024-09-21T10:32:34.844 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -- lvm zap /dev/vg_nvme/lv_2 2024-09-21T10:32:35.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:34 smithi123 ceph-mon[23709]: purged_snaps scrub starts 2024-09-21T10:32:35.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:34 smithi123 ceph-mon[23709]: purged_snaps scrub ok 2024-09-21T10:32:35.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:34 smithi123 ceph-mon[23709]: pgmap v54: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-21T10:32:35.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:34 smithi123 ceph-mon[23709]: osd.1 [v2:172.21.15.123:6810/774521400,v1:172.21.15.123:6811/774521400] boot 2024-09-21T10:32:35.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:34 smithi123 ceph-mon[23709]: osdmap e14: 2 total, 2 up, 2 in 2024-09-21T10:32:35.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:34 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T10:32:35.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:34 smithi175 ceph-mon[25806]: purged_snaps scrub starts 2024-09-21T10:32:35.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:34 smithi175 ceph-mon[25806]: purged_snaps scrub ok 2024-09-21T10:32:35.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:34 smithi175 ceph-mon[25806]: pgmap v54: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-21T10:32:35.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:34 smithi175 ceph-mon[25806]: osd.1 [v2:172.21.15.123:6810/774521400,v1:172.21.15.123:6811/774521400] boot 2024-09-21T10:32:35.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:34 smithi175 ceph-mon[25806]: osdmap e14: 2 total, 2 up, 2 in 2024-09-21T10:32:35.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:34 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T10:32:36.215 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:35 smithi123 ceph-mon[23709]: osdmap e15: 2 total, 2 up, 2 in 2024-09-21T10:32:36.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:35 smithi175 ceph-mon[25806]: osdmap e15: 2 total, 2 up, 2 in 2024-09-21T10:32:37.512 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:37 smithi175 ceph-mon[25806]: pgmap v57: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-21T10:32:37.512 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:37 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:32:37.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:37 smithi123 ceph-mon[23709]: pgmap v57: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-21T10:32:37.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:37 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:32:38.797 INFO:teuthology.orchestra.run.smithi123.stdout: 2024-09-21T10:32:38.819 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -- ceph orch daemon add osd smithi123:vg_nvme/lv_2 2024-09-21T10:32:39.421 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:39 smithi123 ceph-mon[23709]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T10:32:39.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:39 smithi175 ceph-mon[25806]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T10:32:41.393 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:41 smithi123 ceph-mon[23709]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T10:32:41.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:41 smithi175 ceph-mon[25806]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T10:32:42.614 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:42 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:32:42.614 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:42 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T10:32:42.614 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:42 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T10:32:42.614 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:42 smithi123 ceph-mon[23709]: Adjusting osd_memory_target on smithi123 to 6432M 2024-09-21T10:32:42.614 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:42 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:32:42.614 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:42 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:32:42.614 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:42 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:32:42.614 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:42 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:32:42.614 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:42 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T10:32:42.614 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:42 smithi123 ceph-mon[23709]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T10:32:42.614 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:42 smithi123 ceph-mon[23709]: from='client.14268 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi123:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:32:42.614 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:42 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T10:32:42.615 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:42 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T10:32:42.615 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:42 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:32:42.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:42 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:32:42.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:42 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T10:32:42.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:42 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T10:32:42.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:42 smithi175 ceph-mon[25806]: Adjusting osd_memory_target on smithi123 to 6432M 2024-09-21T10:32:42.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:42 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:32:42.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:42 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:32:42.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:42 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:32:42.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:42 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:32:42.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:42 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T10:32:42.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:42 smithi175 ceph-mon[25806]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T10:32:42.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:42 smithi175 ceph-mon[25806]: from='client.14268 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi123:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:32:42.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:42 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T10:32:42.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:42 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T10:32:42.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:42 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:32:44.942 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:44 smithi123 ceph-mon[23709]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T10:32:45.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:44 smithi175 ceph-mon[25806]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T10:32:45.992 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:45 smithi123 ceph-mon[23709]: from='client.? 172.21.15.123:0/1047705887' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3ae9a8ca-f962-4d5b-80e9-ee07fc918c28"}]: dispatch 2024-09-21T10:32:45.992 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:45 smithi123 ceph-mon[23709]: from='client.? 172.21.15.123:0/1047705887' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3ae9a8ca-f962-4d5b-80e9-ee07fc918c28"}]': finished 2024-09-21T10:32:45.992 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:45 smithi123 ceph-mon[23709]: osdmap e16: 3 total, 2 up, 3 in 2024-09-21T10:32:45.992 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:45 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T10:32:46.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:45 smithi175 ceph-mon[25806]: from='client.? 172.21.15.123:0/1047705887' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3ae9a8ca-f962-4d5b-80e9-ee07fc918c28"}]: dispatch 2024-09-21T10:32:46.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:45 smithi175 ceph-mon[25806]: from='client.? 172.21.15.123:0/1047705887' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3ae9a8ca-f962-4d5b-80e9-ee07fc918c28"}]': finished 2024-09-21T10:32:46.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:45 smithi175 ceph-mon[25806]: osdmap e16: 3 total, 2 up, 3 in 2024-09-21T10:32:46.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:45 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T10:32:47.022 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:46 smithi123 ceph-mon[23709]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T10:32:47.022 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:46 smithi123 ceph-mon[23709]: from='client.? 172.21.15.123:0/4126935148' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T10:32:47.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:46 smithi175 ceph-mon[25806]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T10:32:47.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:46 smithi175 ceph-mon[25806]: from='client.? 172.21.15.123:0/4126935148' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T10:32:48.350 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:48 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:32:48.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:48 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:32:48.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:48 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.stnuit/mirror_snapshot_schedule"}]: dispatch 2024-09-21T10:32:48.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:48 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.stnuit/trash_purge_schedule"}]: dispatch 2024-09-21T10:32:48.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:48 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:32:48.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:48 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:32:48.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:48 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.stnuit/mirror_snapshot_schedule"}]: dispatch 2024-09-21T10:32:48.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:48 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.stnuit/trash_purge_schedule"}]: dispatch 2024-09-21T10:32:49.323 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:49 smithi123 ceph-mon[23709]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T10:32:49.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:49 smithi175 ceph-mon[25806]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T10:32:51.221 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:51 smithi123 ceph-mon[23709]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T10:32:51.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:51 smithi175 ceph-mon[25806]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T10:32:52.266 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:52 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-21T10:32:52.266 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:52 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:32:52.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:52 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-21T10:32:52.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:52 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:32:53.160 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:53 smithi123 ceph-mon[23709]: Deploying daemon osd.2 on smithi123 2024-09-21T10:32:53.160 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:53 smithi123 ceph-mon[23709]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T10:32:53.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:53 smithi175 ceph-mon[25806]: Deploying daemon osd.2 on smithi123 2024-09-21T10:32:53.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:53 smithi175 ceph-mon[25806]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T10:32:55.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:55 smithi123 ceph-mon[23709]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T10:32:55.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:55 smithi175 ceph-mon[25806]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T10:32:56.850 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:56 smithi123 ceph-mon[23709]: pgmap v68: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T10:32:56.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:56 smithi175 ceph-mon[25806]: pgmap v68: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T10:32:58.069 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:57 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:32:58.069 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:57 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:32:58.069 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:57 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:32:58.069 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:57 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T10:32:58.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:57 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:32:58.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:57 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:32:58.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:57 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:32:58.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:57 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T10:32:59.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:32:58 smithi123 ceph-mon[23709]: pgmap v69: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T10:32:59.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:32:58 smithi175 ceph-mon[25806]: pgmap v69: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T10:32:59.500 INFO:teuthology.orchestra.run.smithi123.stdout:Created osd(s) 2 on host 'smithi123' 2024-09-21T10:33:00.179 DEBUG:teuthology.orchestra.run.smithi123:osd.2> sudo journalctl -f -n 0 -u ceph-3ccd15e6-7804-11ef-baf6-efdc52797490@osd.2.service 2024-09-21T10:33:00.181 INFO:tasks.cephadm:Deploying osd.3 on smithi123 with /dev/vg_nvme/lv_1... 2024-09-21T10:33:00.181 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -- lvm zap /dev/vg_nvme/lv_1 2024-09-21T10:33:00.773 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:00 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:33:00.773 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:00 smithi123 ceph-mon[23709]: pgmap v70: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T10:33:00.773 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:00 smithi123 ceph-mon[23709]: from='osd.2 [v2:172.21.15.123:6818/398645750,v1:172.21.15.123:6819/398645750]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-21T10:33:00.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:00 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:33:00.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:00 smithi175 ceph-mon[25806]: pgmap v70: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T10:33:00.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:00 smithi175 ceph-mon[25806]: from='osd.2 [v2:172.21.15.123:6818/398645750,v1:172.21.15.123:6819/398645750]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-21T10:33:01.790 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:01 smithi123 ceph-mon[23709]: from='osd.2 [v2:172.21.15.123:6818/398645750,v1:172.21.15.123:6819/398645750]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-21T10:33:01.790 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:01 smithi123 ceph-mon[23709]: osdmap e17: 3 total, 2 up, 3 in 2024-09-21T10:33:01.790 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:01 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T10:33:01.790 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:01 smithi123 ceph-mon[23709]: from='osd.2 [v2:172.21.15.123:6818/398645750,v1:172.21.15.123:6819/398645750]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]: dispatch 2024-09-21T10:33:01.790 INFO:journalctl@ceph.osd.2.smithi123.stdout:Sep 21 10:33:01 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-2[41889]: 2024-09-21T10:33:01.505+0000 7eff0f74d700 -1 osd.2 0 waiting for initial osdmap 2024-09-21T10:33:01.790 INFO:journalctl@ceph.osd.2.smithi123.stdout:Sep 21 10:33:01 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-2[41889]: 2024-09-21T10:33:01.509+0000 7eff080e0700 -1 osd.2 18 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T10:33:01.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:01 smithi175 ceph-mon[25806]: from='osd.2 [v2:172.21.15.123:6818/398645750,v1:172.21.15.123:6819/398645750]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-21T10:33:01.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:01 smithi175 ceph-mon[25806]: osdmap e17: 3 total, 2 up, 3 in 2024-09-21T10:33:01.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:01 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T10:33:01.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:01 smithi175 ceph-mon[25806]: from='osd.2 [v2:172.21.15.123:6818/398645750,v1:172.21.15.123:6819/398645750]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]: dispatch 2024-09-21T10:33:02.699 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:02 smithi123 ceph-mon[23709]: from='osd.2 [v2:172.21.15.123:6818/398645750,v1:172.21.15.123:6819/398645750]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]': finished 2024-09-21T10:33:02.699 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:02 smithi123 ceph-mon[23709]: osdmap e18: 3 total, 2 up, 3 in 2024-09-21T10:33:02.699 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:02 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T10:33:02.699 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:02 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T10:33:02.699 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:02 smithi123 ceph-mon[23709]: pgmap v73: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T10:33:02.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:02 smithi175 ceph-mon[25806]: from='osd.2 [v2:172.21.15.123:6818/398645750,v1:172.21.15.123:6819/398645750]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]': finished 2024-09-21T10:33:02.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:02 smithi175 ceph-mon[25806]: osdmap e18: 3 total, 2 up, 3 in 2024-09-21T10:33:02.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:02 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T10:33:02.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:02 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T10:33:02.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:02 smithi175 ceph-mon[25806]: pgmap v73: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T10:33:03.329 INFO:teuthology.orchestra.run.smithi123.stdout: 2024-09-21T10:33:03.349 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -- ceph orch daemon add osd smithi123:vg_nvme/lv_1 2024-09-21T10:33:03.600 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:03 smithi123 ceph-mon[23709]: purged_snaps scrub starts 2024-09-21T10:33:03.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:03 smithi123 ceph-mon[23709]: purged_snaps scrub ok 2024-09-21T10:33:03.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:03 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T10:33:03.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:03 smithi123 ceph-mon[23709]: osd.2 [v2:172.21.15.123:6818/398645750,v1:172.21.15.123:6819/398645750] boot 2024-09-21T10:33:03.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:03 smithi123 ceph-mon[23709]: osdmap e19: 3 total, 3 up, 3 in 2024-09-21T10:33:03.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:03 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T10:33:03.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:03 smithi175 ceph-mon[25806]: purged_snaps scrub starts 2024-09-21T10:33:03.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:03 smithi175 ceph-mon[25806]: purged_snaps scrub ok 2024-09-21T10:33:03.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:03 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T10:33:03.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:03 smithi175 ceph-mon[25806]: osd.2 [v2:172.21.15.123:6818/398645750,v1:172.21.15.123:6819/398645750] boot 2024-09-21T10:33:03.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:03 smithi175 ceph-mon[25806]: osdmap e19: 3 total, 3 up, 3 in 2024-09-21T10:33:03.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:03 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T10:33:04.850 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:04 smithi123 ceph-mon[23709]: osdmap e20: 3 total, 3 up, 3 in 2024-09-21T10:33:04.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:04 smithi123 ceph-mon[23709]: pgmap v76: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-21T10:33:04.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:04 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-21T10:33:04.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:04 smithi175 ceph-mon[25806]: osdmap e20: 3 total, 3 up, 3 in 2024-09-21T10:33:04.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:04 smithi175 ceph-mon[25806]: pgmap v76: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-21T10:33:04.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:04 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-21T10:33:05.790 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:05 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-21T10:33:05.790 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:05 smithi123 ceph-mon[23709]: osdmap e21: 3 total, 3 up, 3 in 2024-09-21T10:33:05.790 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:05 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-21T10:33:05.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:05 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-21T10:33:05.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:05 smithi175 ceph-mon[25806]: osdmap e21: 3 total, 3 up, 3 in 2024-09-21T10:33:05.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:05 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-21T10:33:06.433 INFO:journalctl@ceph.osd.0.smithi123.stdout:Sep 21 10:33:06 smithi123 sudo[43261]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-21T10:33:06.433 INFO:journalctl@ceph.osd.0.smithi123.stdout:Sep 21 10:33:06 smithi123 sudo[43261]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-21T10:33:06.434 INFO:journalctl@ceph.osd.0.smithi123.stdout:Sep 21 10:33:06 smithi123 sudo[43261]: pam_unix(sudo:session): session closed for user root 2024-09-21T10:33:06.705 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:06 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-21T10:33:06.706 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:06 smithi123 ceph-mon[23709]: osdmap e22: 3 total, 3 up, 3 in 2024-09-21T10:33:06.706 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:06 smithi123 ceph-mon[23709]: pgmap v79: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-21T10:33:06.706 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:06 smithi123 ceph-mon[23709]: from='client.14286 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi123:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:33:06.706 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:06 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T10:33:06.706 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:06 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T10:33:06.706 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:06 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:33:06.706 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:06 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:33:06.706 INFO:journalctl@ceph.osd.0.smithi123.stdout:Sep 21 10:33:06 smithi123 sudo[43333]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-21T10:33:06.706 INFO:journalctl@ceph.osd.0.smithi123.stdout:Sep 21 10:33:06 smithi123 sudo[43333]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-21T10:33:06.706 INFO:journalctl@ceph.osd.0.smithi123.stdout:Sep 21 10:33:06 smithi123 sudo[43333]: pam_unix(sudo:session): session closed for user root 2024-09-21T10:33:06.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:06 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-21T10:33:06.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:06 smithi175 ceph-mon[25806]: osdmap e22: 3 total, 3 up, 3 in 2024-09-21T10:33:06.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:06 smithi175 ceph-mon[25806]: pgmap v79: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-21T10:33:06.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:06 smithi175 ceph-mon[25806]: from='client.14286 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi123:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:33:06.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:06 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T10:33:06.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:06 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T10:33:06.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:06 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:33:06.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:06 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:33:06.963 INFO:journalctl@ceph.osd.1.smithi123.stdout:Sep 21 10:33:06 smithi123 sudo[43353]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-21T10:33:06.963 INFO:journalctl@ceph.osd.1.smithi123.stdout:Sep 21 10:33:06 smithi123 sudo[43353]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-21T10:33:06.963 INFO:journalctl@ceph.osd.1.smithi123.stdout:Sep 21 10:33:06 smithi123 sudo[43353]: pam_unix(sudo:session): session closed for user root 2024-09-21T10:33:06.963 INFO:journalctl@ceph.osd.1.smithi123.stdout:Sep 21 10:33:06 smithi123 sudo[43357]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-21T10:33:07.222 INFO:journalctl@ceph.osd.1.smithi123.stdout:Sep 21 10:33:06 smithi123 sudo[43357]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-21T10:33:07.222 INFO:journalctl@ceph.osd.1.smithi123.stdout:Sep 21 10:33:06 smithi123 sudo[43357]: pam_unix(sudo:session): session closed for user root 2024-09-21T10:33:07.222 INFO:journalctl@ceph.osd.2.smithi123.stdout:Sep 21 10:33:07 smithi123 sudo[43363]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-21T10:33:07.222 INFO:journalctl@ceph.osd.2.smithi123.stdout:Sep 21 10:33:07 smithi123 sudo[43363]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-21T10:33:07.222 INFO:journalctl@ceph.osd.2.smithi123.stdout:Sep 21 10:33:07 smithi123 sudo[43363]: pam_unix(sudo:session): session closed for user root 2024-09-21T10:33:07.600 INFO:journalctl@ceph.osd.2.smithi123.stdout:Sep 21 10:33:07 smithi123 sudo[43366]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-21T10:33:07.600 INFO:journalctl@ceph.osd.2.smithi123.stdout:Sep 21 10:33:07 smithi123 sudo[43366]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-21T10:33:07.600 INFO:journalctl@ceph.osd.2.smithi123.stdout:Sep 21 10:33:07 smithi123 sudo[43366]: pam_unix(sudo:session): session closed for user root 2024-09-21T10:33:07.850 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:07 smithi123 sudo[43398]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-21T10:33:07.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:07 smithi123 sudo[43398]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-21T10:33:07.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:07 smithi123 sudo[43398]: pam_unix(sudo:session): session closed for user root 2024-09-21T10:33:08.243 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:07 smithi175 ceph-mon[25806]: osdmap e23: 3 total, 3 up, 3 in 2024-09-21T10:33:08.243 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:07 smithi175 ceph-mon[25806]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-21T10:33:08.243 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:08 smithi175 sudo[28002]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-21T10:33:08.243 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:08 smithi175 sudo[28002]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-21T10:33:08.243 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:08 smithi175 sudo[28002]: pam_unix(sudo:session): session closed for user root 2024-09-21T10:33:08.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:07 smithi123 sudo[43419]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-21T10:33:08.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:07 smithi123 sudo[43419]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-21T10:33:08.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:07 smithi123 sudo[43419]: pam_unix(sudo:session): session closed for user root 2024-09-21T10:33:08.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:07 smithi123 ceph-mon[23709]: osdmap e23: 3 total, 3 up, 3 in 2024-09-21T10:33:08.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:07 smithi123 ceph-mon[23709]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-21T10:33:08.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:08 smithi175 sudo[28006]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-21T10:33:08.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:08 smithi175 sudo[28006]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-21T10:33:08.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:08 smithi175 sudo[28006]: pam_unix(sudo:session): session closed for user root 2024-09-21T10:33:09.201 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:08 smithi123 ceph-mon[23709]: pgmap v81: 1 pgs: 1 unknown; 0 B data, 16 MiB used, 268 GiB / 268 GiB avail 2024-09-21T10:33:09.201 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:08 smithi123 ceph-mon[23709]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-21T10:33:09.201 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:08 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mon metadata", "id": "smithi123"}]: dispatch 2024-09-21T10:33:09.201 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:08 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mon metadata", "id": "smithi175"}]: dispatch 2024-09-21T10:33:09.201 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:08 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mon metadata", "id": "smithi123"}]: dispatch 2024-09-21T10:33:09.201 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:08 smithi123 ceph-mon[23709]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-21T10:33:09.201 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:08 smithi123 ceph-mon[23709]: mgrmap e19: smithi123.stnuit(active, since 2m), standbys: smithi175.txvwrq 2024-09-21T10:33:09.201 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:08 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mon metadata", "id": "smithi175"}]: dispatch 2024-09-21T10:33:09.201 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:08 smithi123 ceph-mon[23709]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-21T10:33:09.201 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:08 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:33:09.201 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:08 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:33:09.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:08 smithi175 ceph-mon[25806]: pgmap v81: 1 pgs: 1 unknown; 0 B data, 16 MiB used, 268 GiB / 268 GiB avail 2024-09-21T10:33:09.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:08 smithi175 ceph-mon[25806]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-21T10:33:09.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:08 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mon metadata", "id": "smithi123"}]: dispatch 2024-09-21T10:33:09.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:08 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mon metadata", "id": "smithi175"}]: dispatch 2024-09-21T10:33:09.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:08 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mon metadata", "id": "smithi123"}]: dispatch 2024-09-21T10:33:09.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:08 smithi175 ceph-mon[25806]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-21T10:33:09.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:08 smithi175 ceph-mon[25806]: mgrmap e19: smithi123.stnuit(active, since 2m), standbys: smithi175.txvwrq 2024-09-21T10:33:09.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:08 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mon metadata", "id": "smithi175"}]: dispatch 2024-09-21T10:33:09.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:08 smithi175 ceph-mon[25806]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-21T10:33:09.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:08 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:33:09.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:08 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:33:10.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:09 smithi175 ceph-mon[25806]: from='client.? 172.21.15.123:0/913697765' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2d6a21d1-602b-41a3-8cb4-7154605e1390"}]: dispatch 2024-09-21T10:33:10.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:09 smithi175 ceph-mon[25806]: from='client.? 172.21.15.123:0/913697765' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "2d6a21d1-602b-41a3-8cb4-7154605e1390"}]': finished 2024-09-21T10:33:10.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:09 smithi175 ceph-mon[25806]: osdmap e24: 4 total, 3 up, 4 in 2024-09-21T10:33:10.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:09 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T10:33:10.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:09 smithi175 ceph-mon[25806]: from='client.? 172.21.15.123:0/1311079690' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T10:33:10.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:09 smithi123 ceph-mon[23709]: from='client.? 172.21.15.123:0/913697765' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2d6a21d1-602b-41a3-8cb4-7154605e1390"}]: dispatch 2024-09-21T10:33:10.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:09 smithi123 ceph-mon[23709]: from='client.? 172.21.15.123:0/913697765' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "2d6a21d1-602b-41a3-8cb4-7154605e1390"}]': finished 2024-09-21T10:33:10.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:09 smithi123 ceph-mon[23709]: osdmap e24: 4 total, 3 up, 4 in 2024-09-21T10:33:10.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:09 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T10:33:10.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:09 smithi123 ceph-mon[23709]: from='client.? 172.21.15.123:0/1311079690' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T10:33:11.350 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:10 smithi123 ceph-mon[23709]: pgmap v83: 1 pgs: 1 unknown; 0 B data, 17 MiB used, 268 GiB / 268 GiB avail 2024-09-21T10:33:11.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:10 smithi175 ceph-mon[25806]: pgmap v83: 1 pgs: 1 unknown; 0 B data, 17 MiB used, 268 GiB / 268 GiB avail 2024-09-21T10:33:13.269 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:12 smithi123 ceph-mon[23709]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-21T10:33:13.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:12 smithi175 ceph-mon[25806]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-21T10:33:15.265 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:14 smithi123 ceph-mon[23709]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-09-21T10:33:15.349 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:14 smithi175 ceph-mon[25806]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-09-21T10:33:16.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:16 smithi123 ceph-mon[23709]: Detected new or changed devices on smithi123 2024-09-21T10:33:16.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:16 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:33:16.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:16 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T10:33:16.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:16 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T10:33:16.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:16 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T10:33:16.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:16 smithi123 ceph-mon[23709]: Adjusting osd_memory_target on smithi123 to 4288M 2024-09-21T10:33:16.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:16 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:33:16.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:16 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:33:16.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:16 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:33:16.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:16 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:33:16.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:16 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T10:33:16.602 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:16 smithi123 ceph-mon[23709]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-09-21T10:33:16.602 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:16 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:33:16.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:16 smithi175 ceph-mon[25806]: Detected new or changed devices on smithi123 2024-09-21T10:33:16.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:16 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:33:16.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:16 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T10:33:16.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:16 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T10:33:16.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:16 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T10:33:16.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:16 smithi175 ceph-mon[25806]: Adjusting osd_memory_target on smithi123 to 4288M 2024-09-21T10:33:16.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:16 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:33:16.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:16 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:33:16.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:16 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:33:16.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:16 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:33:16.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:16 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T10:33:16.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:16 smithi175 ceph-mon[25806]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-09-21T10:33:16.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:16 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:33:17.509 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:17 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-21T10:33:17.509 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:17 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:33:17.509 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:17 smithi123 ceph-mon[23709]: Deploying daemon osd.3 on smithi123 2024-09-21T10:33:17.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:17 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-21T10:33:17.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:17 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:33:17.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:17 smithi175 ceph-mon[25806]: Deploying daemon osd.3 on smithi123 2024-09-21T10:33:18.433 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:18 smithi123 ceph-mon[23709]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-09-21T10:33:18.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:18 smithi175 ceph-mon[25806]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-09-21T10:33:21.096 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:20 smithi123 ceph-mon[23709]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-09-21T10:33:21.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:20 smithi175 ceph-mon[25806]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-09-21T10:33:23.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:23 smithi123 ceph-mon[23709]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-09-21T10:33:23.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:23 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:33:23.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:23 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:33:23.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:23 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:33:23.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:23 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T10:33:23.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:23 smithi175 ceph-mon[25806]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-09-21T10:33:23.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:23 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:33:23.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:23 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:33:23.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:23 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:33:23.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:23 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T10:33:25.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:25 smithi123 ceph-mon[23709]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-09-21T10:33:25.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:25 smithi175 ceph-mon[25806]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-09-21T10:33:25.816 INFO:teuthology.orchestra.run.smithi123.stdout:Created osd(s) 3 on host 'smithi123' 2024-09-21T10:33:26.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:26 smithi123 ceph-mon[23709]: from='osd.3 [v2:172.21.15.123:6826/780883395,v1:172.21.15.123:6827/780883395]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-21T10:33:26.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:26 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:33:26.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:26 smithi175 ceph-mon[25806]: from='osd.3 [v2:172.21.15.123:6826/780883395,v1:172.21.15.123:6827/780883395]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-21T10:33:26.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:26 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:33:26.479 DEBUG:teuthology.orchestra.run.smithi123:osd.3> sudo journalctl -f -n 0 -u ceph-3ccd15e6-7804-11ef-baf6-efdc52797490@osd.3.service 2024-09-21T10:33:26.481 INFO:tasks.cephadm:Deploying osd.4 on smithi175 with /dev/vg_nvme/lv_4... 2024-09-21T10:33:26.482 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -- lvm zap /dev/vg_nvme/lv_4 2024-09-21T10:33:27.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:27 smithi123 ceph-mon[23709]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-09-21T10:33:27.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:27 smithi123 ceph-mon[23709]: from='osd.3 [v2:172.21.15.123:6826/780883395,v1:172.21.15.123:6827/780883395]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-21T10:33:27.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:27 smithi123 ceph-mon[23709]: osdmap e25: 4 total, 3 up, 4 in 2024-09-21T10:33:27.352 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:27 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T10:33:27.352 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:27 smithi123 ceph-mon[23709]: from='osd.3 [v2:172.21.15.123:6826/780883395,v1:172.21.15.123:6827/780883395]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]: dispatch 2024-09-21T10:33:27.352 INFO:journalctl@ceph.osd.3.smithi123.stdout:Sep 21 10:33:27 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-3[45202]: 2024-09-21T10:33:27.046+0000 7efcffcfa700 -1 osd.3 0 waiting for initial osdmap 2024-09-21T10:33:27.352 INFO:journalctl@ceph.osd.3.smithi123.stdout:Sep 21 10:33:27 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-3[45202]: 2024-09-21T10:33:27.050+0000 7efcfbe94700 -1 osd.3 26 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T10:33:27.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:27 smithi175 ceph-mon[25806]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-09-21T10:33:27.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:27 smithi175 ceph-mon[25806]: from='osd.3 [v2:172.21.15.123:6826/780883395,v1:172.21.15.123:6827/780883395]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-21T10:33:27.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:27 smithi175 ceph-mon[25806]: osdmap e25: 4 total, 3 up, 4 in 2024-09-21T10:33:27.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:27 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T10:33:27.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:27 smithi175 ceph-mon[25806]: from='osd.3 [v2:172.21.15.123:6826/780883395,v1:172.21.15.123:6827/780883395]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]: dispatch 2024-09-21T10:33:28.315 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:28 smithi123 ceph-mon[23709]: from='osd.3 [v2:172.21.15.123:6826/780883395,v1:172.21.15.123:6827/780883395]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]': finished 2024-09-21T10:33:28.315 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:28 smithi123 ceph-mon[23709]: osdmap e26: 4 total, 3 up, 4 in 2024-09-21T10:33:28.315 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:28 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T10:33:28.315 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:28 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T10:33:28.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:28 smithi175 ceph-mon[25806]: from='osd.3 [v2:172.21.15.123:6826/780883395,v1:172.21.15.123:6827/780883395]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi123", "root=default"]}]': finished 2024-09-21T10:33:28.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:28 smithi175 ceph-mon[25806]: osdmap e26: 4 total, 3 up, 4 in 2024-09-21T10:33:28.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:28 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T10:33:28.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:28 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T10:33:29.016 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-09-21T10:33:29.036 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -- ceph orch daemon add osd smithi175:vg_nvme/lv_4 2024-09-21T10:33:29.337 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:29 smithi175 ceph-mon[25806]: purged_snaps scrub starts 2024-09-21T10:33:29.337 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:29 smithi175 ceph-mon[25806]: purged_snaps scrub ok 2024-09-21T10:33:29.337 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:29 smithi175 ceph-mon[25806]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-21T10:33:29.337 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:29 smithi175 ceph-mon[25806]: osd.3 [v2:172.21.15.123:6826/780883395,v1:172.21.15.123:6827/780883395] boot 2024-09-21T10:33:29.337 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:29 smithi175 ceph-mon[25806]: osdmap e27: 4 total, 4 up, 4 in 2024-09-21T10:33:29.337 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:29 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T10:33:29.337 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:29 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:33:29.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:29 smithi123 ceph-mon[23709]: purged_snaps scrub starts 2024-09-21T10:33:29.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:29 smithi123 ceph-mon[23709]: purged_snaps scrub ok 2024-09-21T10:33:29.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:29 smithi123 ceph-mon[23709]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-21T10:33:29.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:29 smithi123 ceph-mon[23709]: osd.3 [v2:172.21.15.123:6826/780883395,v1:172.21.15.123:6827/780883395] boot 2024-09-21T10:33:29.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:29 smithi123 ceph-mon[23709]: osdmap e27: 4 total, 4 up, 4 in 2024-09-21T10:33:29.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:29 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T10:33:29.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:29 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:33:30.333 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:30 smithi123 ceph-mon[23709]: osdmap e28: 4 total, 4 up, 4 in 2024-09-21T10:33:30.336 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:30 smithi175 ceph-mon[25806]: osdmap e28: 4 total, 4 up, 4 in 2024-09-21T10:33:31.241 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:31 smithi175 ceph-mon[25806]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T10:33:31.241 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:31 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T10:33:31.241 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:31 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T10:33:31.241 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:31 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:33:31.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:31 smithi123 ceph-mon[23709]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T10:33:31.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:31 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T10:33:31.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:31 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T10:33:31.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:31 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:33:32.329 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:32 smithi175 ceph-mon[25806]: from='client.24141 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi175:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:33:32.329 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:32 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:33:32.329 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:32 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T10:33:32.329 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:32 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T10:33:32.329 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:32 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T10:33:32.329 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:32 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T10:33:32.329 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:32 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:33:32.329 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:32 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:33:32.329 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:32 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:33:32.329 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:32 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:33:32.329 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:32 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T10:33:32.330 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:32 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:33:32.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:32 smithi123 ceph-mon[23709]: from='client.24141 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi175:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:33:32.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:32 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:33:32.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:32 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T10:33:32.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:32 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T10:33:32.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:32 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T10:33:32.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:32 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T10:33:32.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:32 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:33:32.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:32 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:33:32.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:32 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:33:32.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:32 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:33:32.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:32 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T10:33:32.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:32 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:33:33.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:33 smithi123 ceph-mon[23709]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T10:33:33.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:33 smithi123 ceph-mon[23709]: Adjusting osd_memory_target on smithi123 to 3216M 2024-09-21T10:33:33.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:33 smithi123 ceph-mon[23709]: from='client.? 172.21.15.175:0/4151307635' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5377b10e-ec15-4a23-9d58-67a5582061e3"}]: dispatch 2024-09-21T10:33:33.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:33 smithi123 ceph-mon[23709]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5377b10e-ec15-4a23-9d58-67a5582061e3"}]: dispatch 2024-09-21T10:33:33.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:33 smithi123 ceph-mon[23709]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "5377b10e-ec15-4a23-9d58-67a5582061e3"}]': finished 2024-09-21T10:33:33.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:33 smithi123 ceph-mon[23709]: osdmap e29: 5 total, 4 up, 5 in 2024-09-21T10:33:33.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:33 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T10:33:33.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:33 smithi175 ceph-mon[25806]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T10:33:33.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:33 smithi175 ceph-mon[25806]: Adjusting osd_memory_target on smithi123 to 3216M 2024-09-21T10:33:33.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:33 smithi175 ceph-mon[25806]: from='client.? 172.21.15.175:0/4151307635' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5377b10e-ec15-4a23-9d58-67a5582061e3"}]: dispatch 2024-09-21T10:33:33.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:33 smithi175 ceph-mon[25806]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5377b10e-ec15-4a23-9d58-67a5582061e3"}]: dispatch 2024-09-21T10:33:33.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:33 smithi175 ceph-mon[25806]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "5377b10e-ec15-4a23-9d58-67a5582061e3"}]': finished 2024-09-21T10:33:33.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:33 smithi175 ceph-mon[25806]: osdmap e29: 5 total, 4 up, 5 in 2024-09-21T10:33:33.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:33 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T10:33:34.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:34 smithi123 ceph-mon[23709]: from='client.? 172.21.15.175:0/2608062540' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T10:33:34.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:34 smithi175 ceph-mon[25806]: from='client.? 172.21.15.175:0/2608062540' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T10:33:35.350 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:35 smithi123 ceph-mon[23709]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T10:33:35.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:35 smithi175 ceph-mon[25806]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T10:33:37.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:37 smithi175 ceph-mon[25806]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T10:33:37.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:37 smithi123 ceph-mon[23709]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T10:33:39.350 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:39 smithi123 ceph-mon[23709]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T10:33:39.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:39 smithi175 ceph-mon[25806]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T10:33:40.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:40 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-21T10:33:40.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:40 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:33:40.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:40 smithi123 ceph-mon[23709]: Deploying daemon osd.4 on smithi175 2024-09-21T10:33:40.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:40 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-21T10:33:40.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:40 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:33:40.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:40 smithi175 ceph-mon[25806]: Deploying daemon osd.4 on smithi175 2024-09-21T10:33:41.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:41 smithi123 ceph-mon[23709]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T10:33:41.354 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:41 smithi175 ceph-mon[25806]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T10:33:43.345 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:43 smithi175 ceph-mon[25806]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T10:33:43.350 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:43 smithi123 ceph-mon[23709]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T10:33:45.514 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:45 smithi175 ceph-mon[25806]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T10:33:45.514 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:45 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:33:45.514 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:45 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:33:45.514 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:45 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:33:45.514 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:45 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T10:33:45.600 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:45 smithi123 ceph-mon[23709]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T10:33:45.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:45 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:33:45.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:45 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:33:45.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:45 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:33:45.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:45 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T10:33:47.249 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:47 smithi175 ceph-mon[25806]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T10:33:47.284 INFO:teuthology.orchestra.run.smithi175.stdout:Created osd(s) 4 on host 'smithi175' 2024-09-21T10:33:47.600 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:47 smithi123 ceph-mon[23709]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T10:33:48.532 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:48 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:33:48.532 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:48 smithi175 ceph-mon[25806]: from='osd.4 [v2:172.21.15.175:6800/810564508,v1:172.21.15.175:6801/810564508]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-21T10:33:48.532 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:48 smithi175 ceph-mon[25806]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-21T10:33:48.532 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:48 smithi175 ceph-mon[25806]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T10:33:48.532 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:48 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.stnuit/mirror_snapshot_schedule"}]: dispatch 2024-09-21T10:33:48.533 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:48 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.stnuit/trash_purge_schedule"}]: dispatch 2024-09-21T10:33:48.533 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:48 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:33:48.555 DEBUG:teuthology.orchestra.run.smithi175:osd.4> sudo journalctl -f -n 0 -u ceph-3ccd15e6-7804-11ef-baf6-efdc52797490@osd.4.service 2024-09-21T10:33:48.557 INFO:tasks.cephadm:Deploying osd.5 on smithi175 with /dev/vg_nvme/lv_3... 2024-09-21T10:33:48.557 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -- lvm zap /dev/vg_nvme/lv_3 2024-09-21T10:33:48.602 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:48 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:33:48.602 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:48 smithi123 ceph-mon[23709]: from='osd.4 [v2:172.21.15.175:6800/810564508,v1:172.21.15.175:6801/810564508]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-21T10:33:48.602 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:48 smithi123 ceph-mon[23709]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-21T10:33:48.602 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:48 smithi123 ceph-mon[23709]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T10:33:48.602 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:48 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.stnuit/mirror_snapshot_schedule"}]: dispatch 2024-09-21T10:33:48.602 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:48 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.stnuit/trash_purge_schedule"}]: dispatch 2024-09-21T10:33:48.603 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:48 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:33:49.590 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:49 smithi175 ceph-mon[25806]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-21T10:33:49.590 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:49 smithi175 ceph-mon[25806]: osdmap e30: 5 total, 4 up, 5 in 2024-09-21T10:33:49.590 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:49 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T10:33:49.590 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:49 smithi175 ceph-mon[25806]: from='osd.4 [v2:172.21.15.175:6800/810564508,v1:172.21.15.175:6801/810564508]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-09-21T10:33:49.590 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:49 smithi175 ceph-mon[25806]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-09-21T10:33:49.590 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:33:49 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-4[29923]: 2024-09-21T10:33:49.294+0000 7fd840c61700 -1 osd.4 0 waiting for initial osdmap 2024-09-21T10:33:49.590 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:33:49 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-4[29923]: 2024-09-21T10:33:49.300+0000 7fd83b5f8700 -1 osd.4 31 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T10:33:49.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:49 smithi123 ceph-mon[23709]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-21T10:33:49.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:49 smithi123 ceph-mon[23709]: osdmap e30: 5 total, 4 up, 5 in 2024-09-21T10:33:49.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:49 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T10:33:49.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:49 smithi123 ceph-mon[23709]: from='osd.4 [v2:172.21.15.175:6800/810564508,v1:172.21.15.175:6801/810564508]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-09-21T10:33:49.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:49 smithi123 ceph-mon[23709]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-09-21T10:33:50.463 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:50 smithi175 ceph-mon[25806]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]': finished 2024-09-21T10:33:50.463 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:50 smithi175 ceph-mon[25806]: osdmap e31: 5 total, 4 up, 5 in 2024-09-21T10:33:50.463 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:50 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T10:33:50.463 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:50 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T10:33:50.464 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:50 smithi175 ceph-mon[25806]: pgmap v110: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T10:33:50.600 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:50 smithi123 ceph-mon[23709]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]': finished 2024-09-21T10:33:50.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:50 smithi123 ceph-mon[23709]: osdmap e31: 5 total, 4 up, 5 in 2024-09-21T10:33:50.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:50 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T10:33:50.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:50 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T10:33:50.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:50 smithi123 ceph-mon[23709]: pgmap v110: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T10:33:51.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:51 smithi123 ceph-mon[23709]: purged_snaps scrub starts 2024-09-21T10:33:51.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:51 smithi123 ceph-mon[23709]: purged_snaps scrub ok 2024-09-21T10:33:51.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:51 smithi123 ceph-mon[23709]: osd.4 [v2:172.21.15.175:6800/810564508,v1:172.21.15.175:6801/810564508] boot 2024-09-21T10:33:51.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:51 smithi123 ceph-mon[23709]: osdmap e32: 5 total, 5 up, 5 in 2024-09-21T10:33:51.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:51 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T10:33:51.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:51 smithi123 ceph-mon[23709]: osdmap e33: 5 total, 5 up, 5 in 2024-09-21T10:33:51.618 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:51 smithi175 ceph-mon[25806]: purged_snaps scrub starts 2024-09-21T10:33:51.619 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:51 smithi175 ceph-mon[25806]: purged_snaps scrub ok 2024-09-21T10:33:51.619 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:51 smithi175 ceph-mon[25806]: osd.4 [v2:172.21.15.175:6800/810564508,v1:172.21.15.175:6801/810564508] boot 2024-09-21T10:33:51.619 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:51 smithi175 ceph-mon[25806]: osdmap e32: 5 total, 5 up, 5 in 2024-09-21T10:33:51.619 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:51 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T10:33:51.619 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:51 smithi175 ceph-mon[25806]: osdmap e33: 5 total, 5 up, 5 in 2024-09-21T10:33:52.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:52 smithi175 ceph-mon[25806]: pgmap v113: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-09-21T10:33:52.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:52 smithi175 ceph-mon[25806]: osdmap e34: 5 total, 5 up, 5 in 2024-09-21T10:33:52.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:52 smithi175 ceph-mon[25806]: Detected new or changed devices on smithi175 2024-09-21T10:33:52.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:52 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:33:52.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:52 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T10:33:52.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:52 smithi175 ceph-mon[25806]: Adjusting osd_memory_target on smithi175 to 15936M 2024-09-21T10:33:52.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:52 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:33:52.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:52 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:33:52.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:52 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:33:52.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:52 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:33:52.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:52 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T10:33:52.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:52 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:33:53.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:52 smithi123 ceph-mon[23709]: pgmap v113: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-09-21T10:33:53.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:52 smithi123 ceph-mon[23709]: osdmap e34: 5 total, 5 up, 5 in 2024-09-21T10:33:53.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:52 smithi123 ceph-mon[23709]: Detected new or changed devices on smithi175 2024-09-21T10:33:53.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:52 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:33:53.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:52 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T10:33:53.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:52 smithi123 ceph-mon[23709]: Adjusting osd_memory_target on smithi175 to 15936M 2024-09-21T10:33:53.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:52 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:33:53.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:52 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:33:53.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:52 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:33:53.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:52 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:33:53.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:52 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T10:33:53.102 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:52 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:33:53.332 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-09-21T10:33:53.355 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -- ceph orch daemon add osd smithi175:vg_nvme/lv_3 2024-09-21T10:33:53.656 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:53 smithi175 ceph-mon[25806]: osdmap e35: 5 total, 5 up, 5 in 2024-09-21T10:33:54.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:53 smithi123 ceph-mon[23709]: osdmap e35: 5 total, 5 up, 5 in 2024-09-21T10:33:54.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:54 smithi175 ceph-mon[25806]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-09-21T10:33:54.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:54 smithi175 ceph-mon[25806]: osdmap e36: 5 total, 5 up, 5 in 2024-09-21T10:33:55.100 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:54 smithi123 ceph-mon[23709]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-09-21T10:33:55.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:54 smithi123 ceph-mon[23709]: osdmap e36: 5 total, 5 up, 5 in 2024-09-21T10:33:55.736 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:55 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T10:33:55.737 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:55 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T10:33:55.737 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:55 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:33:56.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:55 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T10:33:56.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:55 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T10:33:56.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:55 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:33:56.659 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:56 smithi175 ceph-mon[25806]: from='client.24159 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi175:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:33:56.659 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:56 smithi175 ceph-mon[25806]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 103 KiB/s, 0 objects/s recovering 2024-09-21T10:33:57.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:56 smithi123 ceph-mon[23709]: from='client.24159 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi175:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:33:57.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:56 smithi123 ceph-mon[23709]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 103 KiB/s, 0 objects/s recovering 2024-09-21T10:33:59.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:58 smithi123 ceph-mon[23709]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-21T10:33:59.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:58 smithi123 ceph-mon[23709]: from='client.? 172.21.15.175:0/669527091' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1c675fbd-f7d6-4535-b420-afaa47e88ad4"}]: dispatch 2024-09-21T10:33:59.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:58 smithi123 ceph-mon[23709]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1c675fbd-f7d6-4535-b420-afaa47e88ad4"}]: dispatch 2024-09-21T10:33:59.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:58 smithi123 ceph-mon[23709]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1c675fbd-f7d6-4535-b420-afaa47e88ad4"}]': finished 2024-09-21T10:33:59.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:58 smithi123 ceph-mon[23709]: osdmap e37: 6 total, 5 up, 6 in 2024-09-21T10:33:59.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:58 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T10:33:59.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:33:58 smithi123 ceph-mon[23709]: from='client.? 172.21.15.175:0/3959866084' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T10:33:59.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:58 smithi175 ceph-mon[25806]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-21T10:33:59.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:58 smithi175 ceph-mon[25806]: from='client.? 172.21.15.175:0/669527091' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1c675fbd-f7d6-4535-b420-afaa47e88ad4"}]: dispatch 2024-09-21T10:33:59.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:58 smithi175 ceph-mon[25806]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1c675fbd-f7d6-4535-b420-afaa47e88ad4"}]: dispatch 2024-09-21T10:33:59.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:58 smithi175 ceph-mon[25806]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1c675fbd-f7d6-4535-b420-afaa47e88ad4"}]': finished 2024-09-21T10:33:59.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:58 smithi175 ceph-mon[25806]: osdmap e37: 6 total, 5 up, 6 in 2024-09-21T10:33:59.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:58 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T10:33:59.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:33:58 smithi175 ceph-mon[25806]: from='client.? 172.21.15.175:0/3959866084' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T10:34:00.833 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:00 smithi175 ceph-mon[25806]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 74 KiB/s, 0 objects/s recovering 2024-09-21T10:34:01.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:00 smithi123 ceph-mon[23709]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 74 KiB/s, 0 objects/s recovering 2024-09-21T10:34:02.860 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:02 smithi175 ceph-mon[25806]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-21T10:34:03.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:02 smithi123 ceph-mon[23709]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-21T10:34:05.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:04 smithi123 ceph-mon[23709]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-21T10:34:05.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:04 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-21T10:34:05.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:04 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:34:05.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:04 smithi123 ceph-mon[23709]: Deploying daemon osd.5 on smithi175 2024-09-21T10:34:05.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:04 smithi175 ceph-mon[25806]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-21T10:34:05.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:04 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-21T10:34:05.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:04 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:34:05.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:04 smithi175 ceph-mon[25806]: Deploying daemon osd.5 on smithi175 2024-09-21T10:34:06.787 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:06 smithi175 ceph-mon[25806]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-21T10:34:07.100 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:06 smithi123 ceph-mon[23709]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-21T10:34:08.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:08 smithi175 ceph-mon[25806]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-21T10:34:09.102 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:08 smithi123 ceph-mon[23709]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-21T10:34:10.453 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:10 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:34:10.453 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:10 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:34:10.453 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:10 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:34:10.453 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:10 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T10:34:10.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:10 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:34:10.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:10 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:34:10.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:10 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:34:10.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:10 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T10:34:11.588 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:11 smithi175 ceph-mon[25806]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-21T10:34:11.600 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:11 smithi123 ceph-mon[23709]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-21T10:34:12.239 INFO:teuthology.orchestra.run.smithi175.stdout:Created osd(s) 5 on host 'smithi175' 2024-09-21T10:34:12.832 DEBUG:teuthology.orchestra.run.smithi175:osd.5> sudo journalctl -f -n 0 -u ceph-3ccd15e6-7804-11ef-baf6-efdc52797490@osd.5.service 2024-09-21T10:34:12.835 INFO:tasks.cephadm:Deploying osd.6 on smithi175 with /dev/vg_nvme/lv_2... 2024-09-21T10:34:12.835 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -- lvm zap /dev/vg_nvme/lv_2 2024-09-21T10:34:13.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:13 smithi175 ceph-mon[25806]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-21T10:34:13.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:13 smithi175 ceph-mon[25806]: from='osd.5 [v2:172.21.15.175:6808/2382596417,v1:172.21.15.175:6809/2382596417]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-21T10:34:13.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:13 smithi175 ceph-mon[25806]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-21T10:34:13.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:13 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:34:13.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:13 smithi123 ceph-mon[23709]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-21T10:34:13.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:13 smithi123 ceph-mon[23709]: from='osd.5 [v2:172.21.15.175:6808/2382596417,v1:172.21.15.175:6809/2382596417]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-21T10:34:13.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:13 smithi123 ceph-mon[23709]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-21T10:34:13.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:13 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:34:14.518 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:14 smithi175 ceph-mon[25806]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-21T10:34:14.518 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:14 smithi175 ceph-mon[25806]: osdmap e38: 6 total, 5 up, 6 in 2024-09-21T10:34:14.518 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:14 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T10:34:14.519 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:14 smithi175 ceph-mon[25806]: from='osd.5 [v2:172.21.15.175:6808/2382596417,v1:172.21.15.175:6809/2382596417]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-09-21T10:34:14.519 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:14 smithi175 ceph-mon[25806]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-09-21T10:34:14.519 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:34:14 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-5[32878]: 2024-09-21T10:34:14.168+0000 7f9187e7b700 -1 osd.5 0 waiting for initial osdmap 2024-09-21T10:34:14.519 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:34:14 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-5[32878]: 2024-09-21T10:34:14.174+0000 7f918080e700 -1 osd.5 39 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T10:34:14.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:14 smithi123 ceph-mon[23709]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-21T10:34:14.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:14 smithi123 ceph-mon[23709]: osdmap e38: 6 total, 5 up, 6 in 2024-09-21T10:34:14.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:14 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T10:34:14.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:14 smithi123 ceph-mon[23709]: from='osd.5 [v2:172.21.15.175:6808/2382596417,v1:172.21.15.175:6809/2382596417]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-09-21T10:34:14.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:14 smithi123 ceph-mon[23709]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-09-21T10:34:15.591 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:15 smithi175 ceph-mon[25806]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-21T10:34:15.591 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:15 smithi175 ceph-mon[25806]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]': finished 2024-09-21T10:34:15.591 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:15 smithi175 ceph-mon[25806]: osdmap e39: 6 total, 5 up, 6 in 2024-09-21T10:34:15.591 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:15 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T10:34:15.591 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:15 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T10:34:15.592 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:15 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:34:15.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:15 smithi123 ceph-mon[23709]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-21T10:34:15.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:15 smithi123 ceph-mon[23709]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]': finished 2024-09-21T10:34:15.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:15 smithi123 ceph-mon[23709]: osdmap e39: 6 total, 5 up, 6 in 2024-09-21T10:34:15.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:15 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T10:34:15.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:15 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T10:34:15.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:15 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:34:15.871 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-09-21T10:34:15.890 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -- ceph orch daemon add osd smithi175:vg_nvme/lv_2 2024-09-21T10:34:16.435 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:16 smithi175 ceph-mon[25806]: purged_snaps scrub starts 2024-09-21T10:34:16.435 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:16 smithi175 ceph-mon[25806]: purged_snaps scrub ok 2024-09-21T10:34:16.435 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:16 smithi175 ceph-mon[25806]: osd.5 [v2:172.21.15.175:6808/2382596417,v1:172.21.15.175:6809/2382596417] boot 2024-09-21T10:34:16.435 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:16 smithi175 ceph-mon[25806]: osdmap e40: 6 total, 6 up, 6 in 2024-09-21T10:34:16.435 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:16 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T10:34:16.435 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:16 smithi175 ceph-mon[25806]: osdmap e41: 6 total, 6 up, 6 in 2024-09-21T10:34:16.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:16 smithi123 ceph-mon[23709]: purged_snaps scrub starts 2024-09-21T10:34:16.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:16 smithi123 ceph-mon[23709]: purged_snaps scrub ok 2024-09-21T10:34:16.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:16 smithi123 ceph-mon[23709]: osd.5 [v2:172.21.15.175:6808/2382596417,v1:172.21.15.175:6809/2382596417] boot 2024-09-21T10:34:16.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:16 smithi123 ceph-mon[23709]: osdmap e40: 6 total, 6 up, 6 in 2024-09-21T10:34:16.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:16 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T10:34:16.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:16 smithi123 ceph-mon[23709]: osdmap e41: 6 total, 6 up, 6 in 2024-09-21T10:34:17.402 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:17 smithi175 ceph-mon[25806]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-21T10:34:17.403 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:17 smithi175 ceph-mon[25806]: osdmap e42: 6 total, 6 up, 6 in 2024-09-21T10:34:17.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:17 smithi123 ceph-mon[23709]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-21T10:34:17.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:17 smithi123 ceph-mon[23709]: osdmap e42: 6 total, 6 up, 6 in 2024-09-21T10:34:18.794 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:18 smithi175 ceph-mon[25806]: pgmap v135: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-21T10:34:18.794 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:18 smithi175 ceph-mon[25806]: osdmap e43: 6 total, 6 up, 6 in 2024-09-21T10:34:19.100 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:18 smithi123 ceph-mon[23709]: pgmap v135: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-21T10:34:19.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:18 smithi123 ceph-mon[23709]: osdmap e43: 6 total, 6 up, 6 in 2024-09-21T10:34:19.699 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:19 smithi175 ceph-mon[25806]: osdmap e44: 6 total, 6 up, 6 in 2024-09-21T10:34:19.699 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:19 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:34:19.699 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:19 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T10:34:19.699 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:19 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T10:34:19.699 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:19 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:34:19.699 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:19 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:34:19.700 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:19 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:34:19.700 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:19 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:34:19.700 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:19 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T10:34:19.764 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:19 smithi123 ceph-mon[23709]: osdmap e44: 6 total, 6 up, 6 in 2024-09-21T10:34:19.764 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:19 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:34:19.765 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:19 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T10:34:19.765 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:19 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T10:34:19.765 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:19 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:34:19.765 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:19 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:34:19.765 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:19 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:34:19.765 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:19 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:34:19.765 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:19 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T10:34:20.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:20 smithi175 ceph-mon[25806]: Detected new or changed devices on smithi175 2024-09-21T10:34:20.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:20 smithi175 ceph-mon[25806]: Adjusting osd_memory_target on smithi175 to 7968M 2024-09-21T10:34:20.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:20 smithi175 ceph-mon[25806]: pgmap v138: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-21T10:34:20.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:20 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:34:20.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:20 smithi175 ceph-mon[25806]: from='client.24177 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi175:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:34:20.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:20 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T10:34:20.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:20 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T10:34:20.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:20 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:34:21.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:20 smithi123 ceph-mon[23709]: Detected new or changed devices on smithi175 2024-09-21T10:34:21.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:20 smithi123 ceph-mon[23709]: Adjusting osd_memory_target on smithi175 to 7968M 2024-09-21T10:34:21.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:20 smithi123 ceph-mon[23709]: pgmap v138: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-21T10:34:21.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:20 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:34:21.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:20 smithi123 ceph-mon[23709]: from='client.24177 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi175:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:34:21.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:20 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T10:34:21.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:20 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T10:34:21.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:20 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:34:23.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:22 smithi123 ceph-mon[23709]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-21T10:34:23.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:22 smithi123 ceph-mon[23709]: from='client.? 172.21.15.175:0/460466555' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "95f278c5-7957-49d9-954a-bbd564ee23f5"}]: dispatch 2024-09-21T10:34:23.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:22 smithi123 ceph-mon[23709]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "95f278c5-7957-49d9-954a-bbd564ee23f5"}]: dispatch 2024-09-21T10:34:23.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:22 smithi123 ceph-mon[23709]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "95f278c5-7957-49d9-954a-bbd564ee23f5"}]': finished 2024-09-21T10:34:23.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:22 smithi123 ceph-mon[23709]: osdmap e45: 7 total, 6 up, 7 in 2024-09-21T10:34:23.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:22 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T10:34:23.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:22 smithi175 ceph-mon[25806]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-21T10:34:23.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:22 smithi175 ceph-mon[25806]: from='client.? 172.21.15.175:0/460466555' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "95f278c5-7957-49d9-954a-bbd564ee23f5"}]: dispatch 2024-09-21T10:34:23.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:22 smithi175 ceph-mon[25806]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "95f278c5-7957-49d9-954a-bbd564ee23f5"}]: dispatch 2024-09-21T10:34:23.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:22 smithi175 ceph-mon[25806]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "95f278c5-7957-49d9-954a-bbd564ee23f5"}]': finished 2024-09-21T10:34:23.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:22 smithi175 ceph-mon[25806]: osdmap e45: 7 total, 6 up, 7 in 2024-09-21T10:34:23.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:22 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T10:34:24.100 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:23 smithi123 ceph-mon[23709]: from='client.? 172.21.15.175:0/1699023245' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T10:34:24.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:23 smithi175 ceph-mon[25806]: from='client.? 172.21.15.175:0/1699023245' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T10:34:25.077 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:24 smithi175 ceph-mon[25806]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-21T10:34:25.100 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:24 smithi123 ceph-mon[23709]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-21T10:34:26.999 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:26 smithi175 ceph-mon[25806]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-21T10:34:27.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:26 smithi123 ceph-mon[23709]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-21T10:34:29.094 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:28 smithi175 ceph-mon[25806]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-21T10:34:29.094 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:28 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-21T10:34:29.095 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:28 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:34:29.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:28 smithi123 ceph-mon[23709]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-21T10:34:29.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:28 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-21T10:34:29.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:28 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:34:30.084 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:29 smithi175 ceph-mon[25806]: Deploying daemon osd.6 on smithi175 2024-09-21T10:34:30.100 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:29 smithi123 ceph-mon[23709]: Deploying daemon osd.6 on smithi175 2024-09-21T10:34:31.077 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:30 smithi175 ceph-mon[25806]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-21T10:34:31.100 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:30 smithi123 ceph-mon[23709]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-21T10:34:32.934 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:32 smithi175 ceph-mon[25806]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-21T10:34:33.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:32 smithi123 ceph-mon[23709]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-21T10:34:34.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:34 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:34:34.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:34 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:34:34.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:34 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:34:34.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:34 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T10:34:34.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:34 smithi123 ceph-mon[23709]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-21T10:34:34.859 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:34 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:34:34.859 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:34 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:34:34.859 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:34 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:34:34.859 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:34 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T10:34:34.859 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:34 smithi175 ceph-mon[25806]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-21T10:34:36.692 INFO:teuthology.orchestra.run.smithi175.stdout:Created osd(s) 6 on host 'smithi175' 2024-09-21T10:34:36.989 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:36 smithi175 ceph-mon[25806]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-21T10:34:36.989 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:36 smithi175 ceph-mon[25806]: from='osd.6 [v2:172.21.15.175:6816/1936422005,v1:172.21.15.175:6817/1936422005]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-21T10:34:36.989 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:36 smithi175 ceph-mon[25806]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-21T10:34:37.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:36 smithi123 ceph-mon[23709]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-21T10:34:37.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:36 smithi123 ceph-mon[23709]: from='osd.6 [v2:172.21.15.175:6816/1936422005,v1:172.21.15.175:6817/1936422005]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-21T10:34:37.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:36 smithi123 ceph-mon[23709]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-21T10:34:37.279 DEBUG:teuthology.orchestra.run.smithi175:osd.6> sudo journalctl -f -n 0 -u ceph-3ccd15e6-7804-11ef-baf6-efdc52797490@osd.6.service 2024-09-21T10:34:37.281 INFO:tasks.cephadm:Deploying osd.7 on smithi175 with /dev/vg_nvme/lv_1... 2024-09-21T10:34:37.281 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -- lvm zap /dev/vg_nvme/lv_1 2024-09-21T10:34:38.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:37 smithi123 ceph-mon[23709]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-21T10:34:38.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:37 smithi123 ceph-mon[23709]: osdmap e46: 7 total, 6 up, 7 in 2024-09-21T10:34:38.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:37 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T10:34:38.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:37 smithi123 ceph-mon[23709]: from='osd.6 [v2:172.21.15.175:6816/1936422005,v1:172.21.15.175:6817/1936422005]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-09-21T10:34:38.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:37 smithi123 ceph-mon[23709]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-09-21T10:34:38.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:37 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:34:38.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:37 smithi175 ceph-mon[25806]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-21T10:34:38.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:37 smithi175 ceph-mon[25806]: osdmap e46: 7 total, 6 up, 7 in 2024-09-21T10:34:38.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:37 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T10:34:38.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:37 smithi175 ceph-mon[25806]: from='osd.6 [v2:172.21.15.175:6816/1936422005,v1:172.21.15.175:6817/1936422005]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-09-21T10:34:38.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:37 smithi175 ceph-mon[25806]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-09-21T10:34:38.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:37 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:34:38.124 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:34:37 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6[35907]: 2024-09-21T10:34:37.679+0000 7f92d5f00700 -1 osd.6 0 waiting for initial osdmap 2024-09-21T10:34:38.124 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:34:37 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6[35907]: 2024-09-21T10:34:37.687+0000 7f92cf895700 -1 osd.6 47 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T10:34:38.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:38 smithi175 ceph-mon[25806]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-21T10:34:38.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:38 smithi175 ceph-mon[25806]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]': finished 2024-09-21T10:34:38.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:38 smithi175 ceph-mon[25806]: osdmap e47: 7 total, 6 up, 7 in 2024-09-21T10:34:38.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:38 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T10:34:38.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:38 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T10:34:39.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:38 smithi123 ceph-mon[23709]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-21T10:34:39.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:38 smithi123 ceph-mon[23709]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]': finished 2024-09-21T10:34:39.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:38 smithi123 ceph-mon[23709]: osdmap e47: 7 total, 6 up, 7 in 2024-09-21T10:34:39.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:38 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T10:34:39.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:38 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T10:34:39.823 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:39 smithi175 ceph-mon[25806]: purged_snaps scrub starts 2024-09-21T10:34:39.823 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:39 smithi175 ceph-mon[25806]: purged_snaps scrub ok 2024-09-21T10:34:39.823 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:39 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T10:34:39.823 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:39 smithi175 ceph-mon[25806]: osd.6 [v2:172.21.15.175:6816/1936422005,v1:172.21.15.175:6817/1936422005] boot 2024-09-21T10:34:39.823 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:39 smithi175 ceph-mon[25806]: osdmap e48: 7 total, 7 up, 7 in 2024-09-21T10:34:39.823 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:39 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T10:34:39.823 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:39 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:34:40.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:39 smithi123 ceph-mon[23709]: purged_snaps scrub starts 2024-09-21T10:34:40.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:39 smithi123 ceph-mon[23709]: purged_snaps scrub ok 2024-09-21T10:34:40.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:39 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T10:34:40.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:39 smithi123 ceph-mon[23709]: osd.6 [v2:172.21.15.175:6816/1936422005,v1:172.21.15.175:6817/1936422005] boot 2024-09-21T10:34:40.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:39 smithi123 ceph-mon[23709]: osdmap e48: 7 total, 7 up, 7 in 2024-09-21T10:34:40.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:39 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T10:34:40.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:39 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:34:40.801 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:40 smithi175 ceph-mon[25806]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-21T10:34:40.801 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:40 smithi175 ceph-mon[25806]: osdmap e49: 7 total, 7 up, 7 in 2024-09-21T10:34:41.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:40 smithi123 ceph-mon[23709]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-21T10:34:41.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:40 smithi123 ceph-mon[23709]: osdmap e49: 7 total, 7 up, 7 in 2024-09-21T10:34:41.560 INFO:teuthology.orchestra.run.smithi175.stdout: 2024-09-21T10:34:41.585 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -- ceph orch daemon add osd smithi175:vg_nvme/lv_1 2024-09-21T10:34:41.815 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:41 smithi175 ceph-mon[25806]: osdmap e50: 7 total, 7 up, 7 in 2024-09-21T10:34:42.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:41 smithi123 ceph-mon[23709]: osdmap e50: 7 total, 7 up, 7 in 2024-09-21T10:34:43.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:42 smithi123 ceph-mon[23709]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-21T10:34:43.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:42 smithi123 ceph-mon[23709]: osdmap e51: 7 total, 7 up, 7 in 2024-09-21T10:34:43.113 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:42 smithi175 ceph-mon[25806]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-21T10:34:43.113 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:42 smithi175 ceph-mon[25806]: osdmap e51: 7 total, 7 up, 7 in 2024-09-21T10:34:43.716 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:43 smithi175 ceph-mon[25806]: osdmap e52: 7 total, 7 up, 7 in 2024-09-21T10:34:44.100 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:43 smithi123 ceph-mon[23709]: osdmap e52: 7 total, 7 up, 7 in 2024-09-21T10:34:45.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:44 smithi123 ceph-mon[23709]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-21T10:34:45.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:44 smithi123 ceph-mon[23709]: from='client.24195 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi175:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:34:45.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:44 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T10:34:45.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:44 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T10:34:45.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:44 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:34:45.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:44 smithi123 ceph-mon[23709]: Detected new or changed devices on smithi175 2024-09-21T10:34:45.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:44 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:34:45.102 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:44 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T10:34:45.102 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:44 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T10:34:45.102 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:44 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T10:34:45.102 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:44 smithi123 ceph-mon[23709]: Adjusting osd_memory_target on smithi175 to 5312M 2024-09-21T10:34:45.102 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:44 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:34:45.102 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:44 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:34:45.102 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:44 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:34:45.102 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:44 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:34:45.102 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:44 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T10:34:45.102 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:44 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:34:45.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:44 smithi175 ceph-mon[25806]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-21T10:34:45.179 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:44 smithi175 ceph-mon[25806]: from='client.24195 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi175:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:34:45.179 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:44 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T10:34:45.179 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:44 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T10:34:45.179 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:44 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:34:45.179 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:44 smithi175 ceph-mon[25806]: Detected new or changed devices on smithi175 2024-09-21T10:34:45.180 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:44 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:34:45.180 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:44 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T10:34:45.180 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:44 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T10:34:45.180 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:44 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T10:34:45.180 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:44 smithi175 ceph-mon[25806]: Adjusting osd_memory_target on smithi175 to 5312M 2024-09-21T10:34:45.180 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:44 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:34:45.180 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:44 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:34:45.180 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:44 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:34:45.180 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:44 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:34:45.180 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:44 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T10:34:45.180 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:44 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:34:47.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:46 smithi123 ceph-mon[23709]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 87 KiB/s, 0 objects/s recovering 2024-09-21T10:34:47.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:46 smithi123 ceph-mon[23709]: from='client.? 172.21.15.175:0/4076964156' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e24733b0-9e7e-4099-9c13-3d2e9b74a3c2"}]: dispatch 2024-09-21T10:34:47.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:46 smithi123 ceph-mon[23709]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e24733b0-9e7e-4099-9c13-3d2e9b74a3c2"}]: dispatch 2024-09-21T10:34:47.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:46 smithi123 ceph-mon[23709]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "e24733b0-9e7e-4099-9c13-3d2e9b74a3c2"}]': finished 2024-09-21T10:34:47.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:46 smithi123 ceph-mon[23709]: osdmap e53: 8 total, 7 up, 8 in 2024-09-21T10:34:47.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:46 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T10:34:47.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:46 smithi175 ceph-mon[25806]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 87 KiB/s, 0 objects/s recovering 2024-09-21T10:34:47.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:46 smithi175 ceph-mon[25806]: from='client.? 172.21.15.175:0/4076964156' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e24733b0-9e7e-4099-9c13-3d2e9b74a3c2"}]: dispatch 2024-09-21T10:34:47.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:46 smithi175 ceph-mon[25806]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e24733b0-9e7e-4099-9c13-3d2e9b74a3c2"}]: dispatch 2024-09-21T10:34:47.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:46 smithi175 ceph-mon[25806]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "e24733b0-9e7e-4099-9c13-3d2e9b74a3c2"}]': finished 2024-09-21T10:34:47.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:46 smithi175 ceph-mon[25806]: osdmap e53: 8 total, 7 up, 8 in 2024-09-21T10:34:47.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:46 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T10:34:48.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:47 smithi123 ceph-mon[23709]: from='client.? 172.21.15.175:0/3419062862' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T10:34:48.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:47 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.stnuit/mirror_snapshot_schedule"}]: dispatch 2024-09-21T10:34:48.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:47 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.stnuit/trash_purge_schedule"}]: dispatch 2024-09-21T10:34:48.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:47 smithi175 ceph-mon[25806]: from='client.? 172.21.15.175:0/3419062862' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T10:34:48.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:47 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.stnuit/mirror_snapshot_schedule"}]: dispatch 2024-09-21T10:34:48.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:47 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.stnuit/trash_purge_schedule"}]: dispatch 2024-09-21T10:34:49.001 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:48 smithi123 ceph-mon[23709]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-21T10:34:49.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:48 smithi175 ceph-mon[25806]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-21T10:34:50.974 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:50 smithi175 ceph-mon[25806]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-21T10:34:51.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:50 smithi123 ceph-mon[23709]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-21T10:34:52.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:52 smithi175 ceph-mon[25806]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-21T10:34:52.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:52 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-21T10:34:52.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:52 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:34:53.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:52 smithi123 ceph-mon[23709]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-21T10:34:53.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:52 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-21T10:34:53.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:52 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:34:54.043 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:53 smithi175 ceph-mon[25806]: Deploying daemon osd.7 on smithi175 2024-09-21T10:34:54.100 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:53 smithi123 ceph-mon[23709]: Deploying daemon osd.7 on smithi175 2024-09-21T10:34:54.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:54 smithi175 ceph-mon[25806]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-21T10:34:55.100 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:54 smithi123 ceph-mon[23709]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-21T10:34:56.992 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:56 smithi175 ceph-mon[25806]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-21T10:34:57.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:56 smithi123 ceph-mon[23709]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-21T10:34:59.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:58 smithi123 ceph-mon[23709]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-21T10:34:59.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:58 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:34:59.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:58 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:34:59.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:58 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:34:59.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:34:58 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T10:34:59.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:58 smithi175 ceph-mon[25806]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-21T10:34:59.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:58 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:34:59.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:58 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:34:59.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:58 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:34:59.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:34:58 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T10:35:00.972 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:00 smithi175 ceph-mon[25806]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-21T10:35:01.001 INFO:teuthology.orchestra.run.smithi175.stdout:Created osd(s) 7 on host 'smithi175' 2024-09-21T10:35:01.100 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:00 smithi123 ceph-mon[23709]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-21T10:35:01.672 DEBUG:teuthology.orchestra.run.smithi175:osd.7> sudo journalctl -f -n 0 -u ceph-3ccd15e6-7804-11ef-baf6-efdc52797490@osd.7.service 2024-09-21T10:35:01.675 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-09-21T10:35:01.675 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -- ceph osd stat -f json 2024-09-21T10:35:01.986 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:01 smithi123 ceph-mon[23709]: from='osd.7 [v2:172.21.15.175:6824/1898750451,v1:172.21.15.175:6825/1898750451]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-21T10:35:01.986 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:01 smithi123 ceph-mon[23709]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-21T10:35:01.986 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:01 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:35:01.993 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:01 smithi175 ceph-mon[25806]: from='osd.7 [v2:172.21.15.175:6824/1898750451,v1:172.21.15.175:6825/1898750451]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-21T10:35:01.993 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:01 smithi175 ceph-mon[25806]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-21T10:35:01.993 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:01 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:35:02.996 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:02 smithi175 ceph-mon[25806]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-21T10:35:02.996 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:02 smithi175 ceph-mon[25806]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-21T10:35:02.996 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:02 smithi175 ceph-mon[25806]: osdmap e54: 8 total, 7 up, 8 in 2024-09-21T10:35:02.996 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:02 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T10:35:02.996 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:02 smithi175 ceph-mon[25806]: from='osd.7 [v2:172.21.15.175:6824/1898750451,v1:172.21.15.175:6825/1898750451]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-09-21T10:35:02.996 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:02 smithi175 ceph-mon[25806]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-09-21T10:35:02.996 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:02 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:35:02.997 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:35:02 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[38891]: 2024-09-21T10:35:02.861+0000 7fbd2454d700 -1 osd.7 0 waiting for initial osdmap 2024-09-21T10:35:02.997 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:35:02 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[38891]: 2024-09-21T10:35:02.869+0000 7fbd1eee4700 -1 osd.7 55 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T10:35:03.067 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:02 smithi123 ceph-mon[23709]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-21T10:35:03.067 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:02 smithi123 ceph-mon[23709]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-21T10:35:03.067 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:02 smithi123 ceph-mon[23709]: osdmap e54: 8 total, 7 up, 8 in 2024-09-21T10:35:03.067 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:02 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T10:35:03.067 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:02 smithi123 ceph-mon[23709]: from='osd.7 [v2:172.21.15.175:6824/1898750451,v1:172.21.15.175:6825/1898750451]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-09-21T10:35:03.067 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:02 smithi123 ceph-mon[23709]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]: dispatch 2024-09-21T10:35:03.068 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:02 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:35:03.741 INFO:teuthology.orchestra.run.smithi123.stdout: 2024-09-21T10:35:04.046 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:03 smithi175 ceph-mon[25806]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]': finished 2024-09-21T10:35:04.046 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:03 smithi175 ceph-mon[25806]: osdmap e55: 8 total, 7 up, 8 in 2024-09-21T10:35:04.046 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:03 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T10:35:04.046 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:03 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T10:35:04.046 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:03 smithi175 ceph-mon[25806]: from='client.? 172.21.15.123:0/2885350602' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-21T10:35:04.352 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:03 smithi123 ceph-mon[23709]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi175", "root=default"]}]': finished 2024-09-21T10:35:04.352 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:03 smithi123 ceph-mon[23709]: osdmap e55: 8 total, 7 up, 8 in 2024-09-21T10:35:04.352 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:03 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T10:35:04.352 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:03 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T10:35:04.352 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:03 smithi123 ceph-mon[23709]: from='client.? 172.21.15.123:0/2885350602' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-21T10:35:04.492 INFO:teuthology.orchestra.run.smithi123.stdout:{"epoch":55,"num_osds":8,"num_up_osds":7,"osd_up_since":1726914878,"num_in_osds":8,"osd_in_since":1726914886,"num_remapped_pgs":0} 2024-09-21T10:35:05.093 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:04 smithi175 ceph-mon[25806]: purged_snaps scrub starts 2024-09-21T10:35:05.094 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:04 smithi175 ceph-mon[25806]: purged_snaps scrub ok 2024-09-21T10:35:05.094 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:04 smithi175 ceph-mon[25806]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-09-21T10:35:05.094 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:04 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T10:35:05.094 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:04 smithi175 ceph-mon[25806]: osd.7 [v2:172.21.15.175:6824/1898750451,v1:172.21.15.175:6825/1898750451] boot 2024-09-21T10:35:05.094 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:04 smithi175 ceph-mon[25806]: osdmap e56: 8 total, 8 up, 8 in 2024-09-21T10:35:05.094 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:04 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T10:35:05.100 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:04 smithi123 ceph-mon[23709]: purged_snaps scrub starts 2024-09-21T10:35:05.100 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:04 smithi123 ceph-mon[23709]: purged_snaps scrub ok 2024-09-21T10:35:05.100 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:04 smithi123 ceph-mon[23709]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-09-21T10:35:05.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:04 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T10:35:05.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:04 smithi123 ceph-mon[23709]: osd.7 [v2:172.21.15.175:6824/1898750451,v1:172.21.15.175:6825/1898750451] boot 2024-09-21T10:35:05.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:04 smithi123 ceph-mon[23709]: osdmap e56: 8 total, 8 up, 8 in 2024-09-21T10:35:05.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:04 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T10:35:05.494 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -- ceph osd stat -f json 2024-09-21T10:35:06.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:05 smithi123 ceph-mon[23709]: osdmap e57: 8 total, 8 up, 8 in 2024-09-21T10:35:06.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:05 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:35:06.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:05 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T10:35:06.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:05 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T10:35:06.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:05 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T10:35:06.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:05 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T10:35:06.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:05 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:35:06.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:05 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:35:06.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:05 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:35:06.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:05 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:35:06.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:05 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T10:35:06.102 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:05 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:35:06.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:05 smithi175 ceph-mon[25806]: osdmap e57: 8 total, 8 up, 8 in 2024-09-21T10:35:06.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:05 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:35:06.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:05 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T10:35:06.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:05 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T10:35:06.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:05 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T10:35:06.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:05 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T10:35:06.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:05 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:35:06.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:05 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:35:06.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:05 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:35:06.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:05 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:35:06.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:05 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T10:35:06.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:05 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:35:07.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:06 smithi123 ceph-mon[23709]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:35:07.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:06 smithi123 ceph-mon[23709]: Detected new or changed devices on smithi175 2024-09-21T10:35:07.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:06 smithi123 ceph-mon[23709]: Adjusting osd_memory_target on smithi175 to 3984M 2024-09-21T10:35:07.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:06 smithi123 ceph-mon[23709]: osdmap e58: 8 total, 8 up, 8 in 2024-09-21T10:35:07.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:06 smithi175 ceph-mon[25806]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:35:07.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:06 smithi175 ceph-mon[25806]: Detected new or changed devices on smithi175 2024-09-21T10:35:07.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:06 smithi175 ceph-mon[25806]: Adjusting osd_memory_target on smithi175 to 3984M 2024-09-21T10:35:07.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:06 smithi175 ceph-mon[25806]: osdmap e58: 8 total, 8 up, 8 in 2024-09-21T10:35:07.512 INFO:teuthology.orchestra.run.smithi123.stdout: 2024-09-21T10:35:08.183 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:07 smithi123 ceph-mon[23709]: osdmap e59: 8 total, 8 up, 8 in 2024-09-21T10:35:08.183 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:07 smithi123 ceph-mon[23709]: from='client.? 172.21.15.123:0/1327544279' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-21T10:35:08.207 INFO:teuthology.orchestra.run.smithi123.stdout:{"epoch":59,"num_osds":8,"num_up_osds":8,"osd_up_since":1726914903,"num_in_osds":8,"osd_in_since":1726914886,"num_remapped_pgs":0} 2024-09-21T10:35:08.207 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -- ceph osd dump --format=json 2024-09-21T10:35:08.275 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:07 smithi175 ceph-mon[25806]: osdmap e59: 8 total, 8 up, 8 in 2024-09-21T10:35:08.275 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:07 smithi175 ceph-mon[25806]: from='client.? 172.21.15.123:0/1327544279' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-21T10:35:08.413 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/mon.smithi123/config 2024-09-21T10:35:09.100 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:08 smithi123 ceph-mon[23709]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:35:09.100 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:08 smithi123 ceph-mon[23709]: osdmap e60: 8 total, 8 up, 8 in 2024-09-21T10:35:09.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:08 smithi175 ceph-mon[25806]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:35:09.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:08 smithi175 ceph-mon[25806]: osdmap e60: 8 total, 8 up, 8 in 2024-09-21T10:35:10.229 INFO:teuthology.orchestra.run.smithi123.stdout: 2024-09-21T10:35:10.229 INFO:teuthology.orchestra.run.smithi123.stdout:{"epoch":60,"fsid":"3ccd15e6-7804-11ef-baf6-efdc52797490","created":"2024-09-21T10:28:55.399906+0000","modified":"2024-09-21T10:35:07.876862+0000","last_up_change":"2024-09-21T10:35:03.857489+0000","last_in_change":"2024-09-21T10:34:46.450485+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T10:33:03.878412+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"23","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"a0c9cae5-8e1a-453f-b682-0b656123004e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6802","nonce":1645528985},{"type":"v1","addr":"172.21.15.123:6803","nonce":1645528985}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6804","nonce":1645528985},{"type":"v1","addr":"172.21.15.123:6805","nonce":1645528985}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6808","nonce":1645528985},{"type":"v1","addr":"172.21.15.123:6809","nonce":1645528985}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6806","nonce":1645528985},{"type":"v1","addr":"172.21.15.123:6807","nonce":1645528985}]},"public_addr":"172.21.15.123:6803/1645528985","cluster_addr":"172.21.15.123:6805/1645528985","heartbeat_back_addr":"172.21.15.123:6809/1645528985","heartbeat_front_addr":"172.21.15.123:6807/1645528985","state":["exists","up"]},{"osd":1,"uuid":"f686f59d-6dbf-47c4-8ff0-3f2dbf64e1e1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":35,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6810","nonce":774521400},{"type":"v1","addr":"172.21.15.123:6811","nonce":774521400}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6812","nonce":774521400},{"type":"v1","addr":"172.21.15.123:6813","nonce":774521400}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6816","nonce":774521400},{"type":"v1","addr":"172.21.15.123:6817","nonce":774521400}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6814","nonce":774521400},{"type":"v1","addr":"172.21.15.123:6815","nonce":774521400}]},"public_addr":"172.21.15.123:6811/774521400","cluster_addr":"172.21.15.123:6813/774521400","heartbeat_back_addr":"172.21.15.123:6817/774521400","heartbeat_front_addr":"172.21.15.123:6815/774521400","state":["exists","up"]},{"osd":2,"uuid":"3ae9a8ca-f962-4d5b-80e9-ee07fc918c28","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6818","nonce":398645750},{"type":"v1","addr":"172.21.15.123:6819","nonce":398645750}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6820","nonce":398645750},{"type":"v1","addr":"172.21.15.123:6821","nonce":398645750}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6824","nonce":398645750},{"type":"v1","addr":"172.21.15.123:6825","nonce":398645750}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6822","nonce":398645750},{"type":"v1","addr":"172.21.15.123:6823","nonce":398645750}]},"public_addr":"172.21.15.123:6819/398645750","cluster_addr":"172.21.15.123:6821/398645750","heartbeat_back_addr":"172.21.15.123:6825/398645750","heartbeat_front_addr":"172.21.15.123:6823/398645750","state":["exists","up"]},{"osd":3,"uuid":"2d6a21d1-602b-41a3-8cb4-7154605e1390","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":27,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6826","nonce":780883395},{"type":"v1","addr":"172.21.15.123:6827","nonce":780883395}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6828","nonce":780883395},{"type":"v1","addr":"172.21.15.123:6829","nonce":780883395}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6832","nonce":780883395},{"type":"v1","addr":"172.21.15.123:6833","nonce":780883395}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6830","nonce":780883395},{"type":"v1","addr":"172.21.15.123:6831","nonce":780883395}]},"public_addr":"172.21.15.123:6827/780883395","cluster_addr":"172.21.15.123:6829/780883395","heartbeat_back_addr":"172.21.15.123:6833/780883395","heartbeat_front_addr":"172.21.15.123:6831/780883395","state":["exists","up"]},{"osd":4,"uuid":"5377b10e-ec15-4a23-9d58-67a5582061e3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":32,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6800","nonce":810564508},{"type":"v1","addr":"172.21.15.175:6801","nonce":810564508}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6802","nonce":810564508},{"type":"v1","addr":"172.21.15.175:6803","nonce":810564508}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6806","nonce":810564508},{"type":"v1","addr":"172.21.15.175:6807","nonce":810564508}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6804","nonce":810564508},{"type":"v1","addr":"172.21.15.175:6805","nonce":810564508}]},"public_addr":"172.21.15.175:6801/810564508","cluster_addr":"172.21.15.175:6803/810564508","heartbeat_back_addr":"172.21.15.175:6807/810564508","heartbeat_front_addr":"172.21.15.175:6805/810564508","state":["exists","up"]},{"osd":5,"uuid":"1c675fbd-f7d6-4535-b420-afaa47e88ad4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6808","nonce":2382596417},{"type":"v1","addr":"172.21.15.175:6809","nonce":2382596417}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6810","nonce":2382596417},{"type":"v1","addr":"172.21.15.175:6811","nonce":2382596417}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6814","nonce":2382596417},{"type":"v1","addr":"172.21.15.175:6815","nonce":2382596417}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6812","nonce":2382596417},{"type":"v1","addr":"172.21.15.175:6813","nonce":2382596417}]},"public_addr":"172.21.15.175:6809/2382596417","cluster_addr":"172.21.15.175:6811/2382596417","heartbeat_back_addr":"172.21.15.175:6815/2382596417","heartbeat_front_addr":"172.21.15.175:6813/2382596417","state":["exists","up"]},{"osd":6,"uuid":"95f278c5-7957-49d9-954a-bbd564ee23f5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":51,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6816","nonce":1936422005},{"type":"v1","addr":"172.21.15.175:6817","nonce":1936422005}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6818","nonce":1936422005},{"type":"v1","addr":"172.21.15.175:6819","nonce":1936422005}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6822","nonce":1936422005},{"type":"v1","addr":"172.21.15.175:6823","nonce":1936422005}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6820","nonce":1936422005},{"type":"v1","addr":"172.21.15.175:6821","nonce":1936422005}]},"public_addr":"172.21.15.175:6817/1936422005","cluster_addr":"172.21.15.175:6819/1936422005","heartbeat_back_addr":"172.21.15.175:6823/1936422005","heartbeat_front_addr":"172.21.15.175:6821/1936422005","state":["exists","up"]},{"osd":7,"uuid":"e24733b0-9e7e-4099-9c13-3d2e9b74a3c2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":56,"up_thru":59,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6824","nonce":1898750451},{"type":"v1","addr":"172.21.15.175:6825","nonce":1898750451}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6826","nonce":1898750451},{"type":"v1","addr":"172.21.15.175:6827","nonce":1898750451}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6830","nonce":1898750451},{"type":"v1","addr":"172.21.15.175:6831","nonce":1898750451}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6828","nonce":1898750451},{"type":"v1","addr":"172.21.15.175:6829","nonce":1898750451}]},"public_addr":"172.21.15.175:6825/1898750451","cluster_addr":"172.21.15.175:6827/1898750451","heartbeat_back_addr":"172.21.15.175:6831/1898750451","heartbeat_front_addr":"172.21.15.175:6829/1898750451","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T10:32:00.399459+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T10:32:32.972827+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T10:33:00.830217+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T10:33:26.137100+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T10:33:48.340351+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T10:34:13.218938+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T10:34:37.659752+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T10:35:01.976363+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.123:0/2332584426":"2024-09-22T10:30:47.165904+0000","172.21.15.123:6801/1080622549":"2024-09-22T10:30:47.165904+0000","172.21.15.123:0/2795289521":"2024-09-22T10:30:47.165904+0000","172.21.15.123:6800/1080622549":"2024-09-22T10:30:47.165904+0000","172.21.15.123:0/1044945966":"2024-09-22T10:30:47.165904+0000","172.21.15.123:6800/1777975386":"2024-09-22T10:29:57.788390+0000","172.21.15.123:0/2899987542":"2024-09-22T10:29:57.788390+0000","172.21.15.123:0/1641759314":"2024-09-22T10:29:57.788390+0000","172.21.15.123:0/1586135573":"2024-09-22T10:29:57.788390+0000","172.21.15.123:0/4085669181":"2024-09-22T10:29:27.633648+0000","172.21.15.123:6801/1777975386":"2024-09-22T10:29:57.788390+0000","172.21.15.123:6800/306380447":"2024-09-22T10:29:27.633648+0000","172.21.15.123:0/3594083655":"2024-09-22T10:29:27.633648+0000","172.21.15.123:6801/306380447":"2024-09-22T10:29:27.633648+0000","172.21.15.123:0/2562221798":"2024-09-22T10:29:27.633648+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T10:35:10.929 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-09-21T10:33:03.878412+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '23', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-09-21T10:35:10.930 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -- ceph osd pool get .mgr pg_num 2024-09-21T10:35:11.134 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/mon.smithi123/config 2024-09-21T10:35:11.255 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:10 smithi123 ceph-mon[23709]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:35:11.255 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:10 smithi123 ceph-mon[23709]: from='client.? 172.21.15.123:0/317969893' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T10:35:11.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:10 smithi175 ceph-mon[25806]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:35:11.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:10 smithi175 ceph-mon[25806]: from='client.? 172.21.15.123:0/317969893' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T10:35:12.932 INFO:teuthology.orchestra.run.smithi123.stdout:pg_num: 1 2024-09-21T10:35:12.933 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:12 smithi123 ceph-mon[23709]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-21T10:35:13.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:12 smithi175 ceph-mon[25806]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-21T10:35:13.723 INFO:tasks.cephadm:Setting up client nodes... 2024-09-21T10:35:13.724 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-09-21T10:35:14.018 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:13 smithi123 ceph-mon[23709]: from='client.? 172.21.15.123:0/3896606428' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-21T10:35:14.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:13 smithi175 ceph-mon[25806]: from='client.? 172.21.15.123:0/3896606428' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-21T10:35:15.173 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:14 smithi123 ceph-mon[23709]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 67 KiB/s, 0 objects/s recovering 2024-09-21T10:35:15.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:14 smithi175 ceph-mon[25806]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 67 KiB/s, 0 objects/s recovering 2024-09-21T10:35:15.722 INFO:teuthology.orchestra.run.smithi123.stdout:[client.0] 2024-09-21T10:35:15.723 INFO:teuthology.orchestra.run.smithi123.stdout: key = AQBjoe5meC/CKhAAvEmz9gysl1av6ND5qWWlLw== 2024-09-21T10:35:16.481 DEBUG:teuthology.orchestra.run.smithi123:> set -ex 2024-09-21T10:35:16.481 DEBUG:teuthology.orchestra.run.smithi123:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-09-21T10:35:16.481 DEBUG:teuthology.orchestra.run.smithi123:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-09-21T10:35:16.517 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -- ceph config log 1 --format=json 2024-09-21T10:35:16.762 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/mon.smithi123/config 2024-09-21T10:35:16.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:16 smithi175 ceph-mon[25806]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 59 KiB/s, 0 objects/s recovering 2024-09-21T10:35:16.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:16 smithi175 ceph-mon[25806]: from='client.? 172.21.15.123:0/3635094890' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-21T10:35:16.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:16 smithi175 ceph-mon[25806]: from='client.? 172.21.15.123:0/3635094890' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-21T10:35:16.882 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:16 smithi123 ceph-mon[23709]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 59 KiB/s, 0 objects/s recovering 2024-09-21T10:35:16.882 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:16 smithi123 ceph-mon[23709]: from='client.? 172.21.15.123:0/3635094890' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-21T10:35:16.883 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:16 smithi123 ceph-mon[23709]: from='client.? 172.21.15.123:0/3635094890' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-21T10:35:18.504 INFO:teuthology.orchestra.run.smithi123.stdout: 2024-09-21T10:35:18.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:18 smithi123 ceph-mon[23709]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-21T10:35:18.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:18 smithi123 ceph-mon[23709]: from='client.? 172.21.15.123:0/1393031663' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-09-21T10:35:19.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:18 smithi175 ceph-mon[25806]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-21T10:35:19.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:18 smithi175 ceph-mon[25806]: from='client.? 172.21.15.123:0/1393031663' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-09-21T10:35:19.155 INFO:teuthology.orchestra.run.smithi123.stdout:[{"version":21,"timestamp":"2024-09-21T10:35:05.773925+0000","name":"","changes":[{"name":"osd/host:smithi175/osd_memory_target","previous_value":"5570054963","new_value":"4177541222"}]}] 2024-09-21T10:35:19.155 INFO:tasks.ceph_manager:config epoch is 21 2024-09-21T10:35:19.155 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-09-21T10:35:19.156 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-09-21T10:35:19.156 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -- ceph mgr dump --format=json 2024-09-21T10:35:19.360 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/mon.smithi123/config 2024-09-21T10:35:20.772 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:20 smithi123 ceph-mon[23709]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 44 KiB/s, 0 objects/s recovering 2024-09-21T10:35:21.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:20 smithi175 ceph-mon[25806]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 44 KiB/s, 0 objects/s recovering 2024-09-21T10:35:21.242 INFO:teuthology.orchestra.run.smithi123.stdout: 2024-09-21T10:35:21.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:21 smithi123 ceph-mon[23709]: from='client.? 172.21.15.123:0/658052094' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-21T10:35:21.875 INFO:teuthology.orchestra.run.smithi123.stdout:{"epoch":19,"active_gid":14202,"active_name":"smithi123.stnuit","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6800","nonce":495388425},{"type":"v1","addr":"172.21.15.123:6801","nonce":495388425}]},"active_addr":"172.21.15.123:6801/495388425","active_change":"2024-09-21T10:30:47.166203+0000","active_mgr_features":4540138303579357183,"available":true,"standbys":[{"gid":14222,"name":"smithi175.txvwrq","mgr_features":4540138303579357183,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.123:8443/","prometheus":"http://172.21.15.123:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"last_failure_osd_epoch":5,"active_clients":[{"addrvec":[{"type":"v2","addr":"172.21.15.123:0","nonce":1336217104}]},{"addrvec":[{"type":"v2","addr":"172.21.15.123:0","nonce":505506975}]},{"addrvec":[{"type":"v2","addr":"172.21.15.123:0","nonce":4205936143}]},{"addrvec":[{"type":"v2","addr":"172.21.15.123:0","nonce":3652770252}]}]}} 2024-09-21T10:35:21.878 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-09-21T10:35:21.878 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-09-21T10:35:21.879 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -- ceph osd dump --format=json 2024-09-21T10:35:22.089 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/mon.smithi123/config 2024-09-21T10:35:22.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:21 smithi175 ceph-mon[25806]: from='client.? 172.21.15.123:0/658052094' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-21T10:35:22.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:22 smithi123 ceph-mon[23709]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s, 0 objects/s recovering 2024-09-21T10:35:23.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:22 smithi175 ceph-mon[25806]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s, 0 objects/s recovering 2024-09-21T10:35:23.882 INFO:teuthology.orchestra.run.smithi123.stdout: 2024-09-21T10:35:23.882 INFO:teuthology.orchestra.run.smithi123.stdout:{"epoch":60,"fsid":"3ccd15e6-7804-11ef-baf6-efdc52797490","created":"2024-09-21T10:28:55.399906+0000","modified":"2024-09-21T10:35:07.876862+0000","last_up_change":"2024-09-21T10:35:03.857489+0000","last_in_change":"2024-09-21T10:34:46.450485+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T10:33:03.878412+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"23","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"a0c9cae5-8e1a-453f-b682-0b656123004e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6802","nonce":1645528985},{"type":"v1","addr":"172.21.15.123:6803","nonce":1645528985}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6804","nonce":1645528985},{"type":"v1","addr":"172.21.15.123:6805","nonce":1645528985}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6808","nonce":1645528985},{"type":"v1","addr":"172.21.15.123:6809","nonce":1645528985}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6806","nonce":1645528985},{"type":"v1","addr":"172.21.15.123:6807","nonce":1645528985}]},"public_addr":"172.21.15.123:6803/1645528985","cluster_addr":"172.21.15.123:6805/1645528985","heartbeat_back_addr":"172.21.15.123:6809/1645528985","heartbeat_front_addr":"172.21.15.123:6807/1645528985","state":["exists","up"]},{"osd":1,"uuid":"f686f59d-6dbf-47c4-8ff0-3f2dbf64e1e1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":35,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6810","nonce":774521400},{"type":"v1","addr":"172.21.15.123:6811","nonce":774521400}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6812","nonce":774521400},{"type":"v1","addr":"172.21.15.123:6813","nonce":774521400}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6816","nonce":774521400},{"type":"v1","addr":"172.21.15.123:6817","nonce":774521400}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6814","nonce":774521400},{"type":"v1","addr":"172.21.15.123:6815","nonce":774521400}]},"public_addr":"172.21.15.123:6811/774521400","cluster_addr":"172.21.15.123:6813/774521400","heartbeat_back_addr":"172.21.15.123:6817/774521400","heartbeat_front_addr":"172.21.15.123:6815/774521400","state":["exists","up"]},{"osd":2,"uuid":"3ae9a8ca-f962-4d5b-80e9-ee07fc918c28","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6818","nonce":398645750},{"type":"v1","addr":"172.21.15.123:6819","nonce":398645750}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6820","nonce":398645750},{"type":"v1","addr":"172.21.15.123:6821","nonce":398645750}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6824","nonce":398645750},{"type":"v1","addr":"172.21.15.123:6825","nonce":398645750}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6822","nonce":398645750},{"type":"v1","addr":"172.21.15.123:6823","nonce":398645750}]},"public_addr":"172.21.15.123:6819/398645750","cluster_addr":"172.21.15.123:6821/398645750","heartbeat_back_addr":"172.21.15.123:6825/398645750","heartbeat_front_addr":"172.21.15.123:6823/398645750","state":["exists","up"]},{"osd":3,"uuid":"2d6a21d1-602b-41a3-8cb4-7154605e1390","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":27,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6826","nonce":780883395},{"type":"v1","addr":"172.21.15.123:6827","nonce":780883395}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6828","nonce":780883395},{"type":"v1","addr":"172.21.15.123:6829","nonce":780883395}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6832","nonce":780883395},{"type":"v1","addr":"172.21.15.123:6833","nonce":780883395}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6830","nonce":780883395},{"type":"v1","addr":"172.21.15.123:6831","nonce":780883395}]},"public_addr":"172.21.15.123:6827/780883395","cluster_addr":"172.21.15.123:6829/780883395","heartbeat_back_addr":"172.21.15.123:6833/780883395","heartbeat_front_addr":"172.21.15.123:6831/780883395","state":["exists","up"]},{"osd":4,"uuid":"5377b10e-ec15-4a23-9d58-67a5582061e3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":32,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6800","nonce":810564508},{"type":"v1","addr":"172.21.15.175:6801","nonce":810564508}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6802","nonce":810564508},{"type":"v1","addr":"172.21.15.175:6803","nonce":810564508}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6806","nonce":810564508},{"type":"v1","addr":"172.21.15.175:6807","nonce":810564508}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6804","nonce":810564508},{"type":"v1","addr":"172.21.15.175:6805","nonce":810564508}]},"public_addr":"172.21.15.175:6801/810564508","cluster_addr":"172.21.15.175:6803/810564508","heartbeat_back_addr":"172.21.15.175:6807/810564508","heartbeat_front_addr":"172.21.15.175:6805/810564508","state":["exists","up"]},{"osd":5,"uuid":"1c675fbd-f7d6-4535-b420-afaa47e88ad4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6808","nonce":2382596417},{"type":"v1","addr":"172.21.15.175:6809","nonce":2382596417}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6810","nonce":2382596417},{"type":"v1","addr":"172.21.15.175:6811","nonce":2382596417}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6814","nonce":2382596417},{"type":"v1","addr":"172.21.15.175:6815","nonce":2382596417}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6812","nonce":2382596417},{"type":"v1","addr":"172.21.15.175:6813","nonce":2382596417}]},"public_addr":"172.21.15.175:6809/2382596417","cluster_addr":"172.21.15.175:6811/2382596417","heartbeat_back_addr":"172.21.15.175:6815/2382596417","heartbeat_front_addr":"172.21.15.175:6813/2382596417","state":["exists","up"]},{"osd":6,"uuid":"95f278c5-7957-49d9-954a-bbd564ee23f5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":51,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6816","nonce":1936422005},{"type":"v1","addr":"172.21.15.175:6817","nonce":1936422005}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6818","nonce":1936422005},{"type":"v1","addr":"172.21.15.175:6819","nonce":1936422005}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6822","nonce":1936422005},{"type":"v1","addr":"172.21.15.175:6823","nonce":1936422005}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6820","nonce":1936422005},{"type":"v1","addr":"172.21.15.175:6821","nonce":1936422005}]},"public_addr":"172.21.15.175:6817/1936422005","cluster_addr":"172.21.15.175:6819/1936422005","heartbeat_back_addr":"172.21.15.175:6823/1936422005","heartbeat_front_addr":"172.21.15.175:6821/1936422005","state":["exists","up"]},{"osd":7,"uuid":"e24733b0-9e7e-4099-9c13-3d2e9b74a3c2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":56,"up_thru":59,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6824","nonce":1898750451},{"type":"v1","addr":"172.21.15.175:6825","nonce":1898750451}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6826","nonce":1898750451},{"type":"v1","addr":"172.21.15.175:6827","nonce":1898750451}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6830","nonce":1898750451},{"type":"v1","addr":"172.21.15.175:6831","nonce":1898750451}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6828","nonce":1898750451},{"type":"v1","addr":"172.21.15.175:6829","nonce":1898750451}]},"public_addr":"172.21.15.175:6825/1898750451","cluster_addr":"172.21.15.175:6827/1898750451","heartbeat_back_addr":"172.21.15.175:6831/1898750451","heartbeat_front_addr":"172.21.15.175:6829/1898750451","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T10:32:00.399459+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T10:32:32.972827+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T10:33:00.830217+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T10:33:26.137100+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T10:33:48.340351+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T10:34:13.218938+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T10:34:37.659752+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T10:35:01.976363+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.123:0/2332584426":"2024-09-22T10:30:47.165904+0000","172.21.15.123:6801/1080622549":"2024-09-22T10:30:47.165904+0000","172.21.15.123:0/2795289521":"2024-09-22T10:30:47.165904+0000","172.21.15.123:6800/1080622549":"2024-09-22T10:30:47.165904+0000","172.21.15.123:0/1044945966":"2024-09-22T10:30:47.165904+0000","172.21.15.123:6800/1777975386":"2024-09-22T10:29:57.788390+0000","172.21.15.123:0/2899987542":"2024-09-22T10:29:57.788390+0000","172.21.15.123:0/1641759314":"2024-09-22T10:29:57.788390+0000","172.21.15.123:0/1586135573":"2024-09-22T10:29:57.788390+0000","172.21.15.123:0/4085669181":"2024-09-22T10:29:27.633648+0000","172.21.15.123:6801/1777975386":"2024-09-22T10:29:57.788390+0000","172.21.15.123:6800/306380447":"2024-09-22T10:29:27.633648+0000","172.21.15.123:0/3594083655":"2024-09-22T10:29:27.633648+0000","172.21.15.123:6801/306380447":"2024-09-22T10:29:27.633648+0000","172.21.15.123:0/2562221798":"2024-09-22T10:29:27.633648+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T10:35:24.666 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-09-21T10:35:24.666 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -- ceph osd dump --format=json 2024-09-21T10:35:24.873 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/mon.smithi123/config 2024-09-21T10:35:24.994 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:24 smithi123 ceph-mon[23709]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:35:24.994 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:24 smithi123 ceph-mon[23709]: from='client.? 172.21.15.123:0/2250452162' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T10:35:25.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:24 smithi175 ceph-mon[25806]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:35:25.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:24 smithi175 ceph-mon[25806]: from='client.? 172.21.15.123:0/2250452162' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T10:35:26.612 INFO:teuthology.orchestra.run.smithi123.stdout: 2024-09-21T10:35:26.613 INFO:teuthology.orchestra.run.smithi123.stdout:{"epoch":60,"fsid":"3ccd15e6-7804-11ef-baf6-efdc52797490","created":"2024-09-21T10:28:55.399906+0000","modified":"2024-09-21T10:35:07.876862+0000","last_up_change":"2024-09-21T10:35:03.857489+0000","last_in_change":"2024-09-21T10:34:46.450485+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T10:33:03.878412+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"23","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"a0c9cae5-8e1a-453f-b682-0b656123004e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6802","nonce":1645528985},{"type":"v1","addr":"172.21.15.123:6803","nonce":1645528985}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6804","nonce":1645528985},{"type":"v1","addr":"172.21.15.123:6805","nonce":1645528985}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6808","nonce":1645528985},{"type":"v1","addr":"172.21.15.123:6809","nonce":1645528985}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6806","nonce":1645528985},{"type":"v1","addr":"172.21.15.123:6807","nonce":1645528985}]},"public_addr":"172.21.15.123:6803/1645528985","cluster_addr":"172.21.15.123:6805/1645528985","heartbeat_back_addr":"172.21.15.123:6809/1645528985","heartbeat_front_addr":"172.21.15.123:6807/1645528985","state":["exists","up"]},{"osd":1,"uuid":"f686f59d-6dbf-47c4-8ff0-3f2dbf64e1e1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":35,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6810","nonce":774521400},{"type":"v1","addr":"172.21.15.123:6811","nonce":774521400}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6812","nonce":774521400},{"type":"v1","addr":"172.21.15.123:6813","nonce":774521400}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6816","nonce":774521400},{"type":"v1","addr":"172.21.15.123:6817","nonce":774521400}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6814","nonce":774521400},{"type":"v1","addr":"172.21.15.123:6815","nonce":774521400}]},"public_addr":"172.21.15.123:6811/774521400","cluster_addr":"172.21.15.123:6813/774521400","heartbeat_back_addr":"172.21.15.123:6817/774521400","heartbeat_front_addr":"172.21.15.123:6815/774521400","state":["exists","up"]},{"osd":2,"uuid":"3ae9a8ca-f962-4d5b-80e9-ee07fc918c28","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6818","nonce":398645750},{"type":"v1","addr":"172.21.15.123:6819","nonce":398645750}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6820","nonce":398645750},{"type":"v1","addr":"172.21.15.123:6821","nonce":398645750}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6824","nonce":398645750},{"type":"v1","addr":"172.21.15.123:6825","nonce":398645750}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6822","nonce":398645750},{"type":"v1","addr":"172.21.15.123:6823","nonce":398645750}]},"public_addr":"172.21.15.123:6819/398645750","cluster_addr":"172.21.15.123:6821/398645750","heartbeat_back_addr":"172.21.15.123:6825/398645750","heartbeat_front_addr":"172.21.15.123:6823/398645750","state":["exists","up"]},{"osd":3,"uuid":"2d6a21d1-602b-41a3-8cb4-7154605e1390","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":27,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6826","nonce":780883395},{"type":"v1","addr":"172.21.15.123:6827","nonce":780883395}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6828","nonce":780883395},{"type":"v1","addr":"172.21.15.123:6829","nonce":780883395}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6832","nonce":780883395},{"type":"v1","addr":"172.21.15.123:6833","nonce":780883395}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.123:6830","nonce":780883395},{"type":"v1","addr":"172.21.15.123:6831","nonce":780883395}]},"public_addr":"172.21.15.123:6827/780883395","cluster_addr":"172.21.15.123:6829/780883395","heartbeat_back_addr":"172.21.15.123:6833/780883395","heartbeat_front_addr":"172.21.15.123:6831/780883395","state":["exists","up"]},{"osd":4,"uuid":"5377b10e-ec15-4a23-9d58-67a5582061e3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":32,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6800","nonce":810564508},{"type":"v1","addr":"172.21.15.175:6801","nonce":810564508}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6802","nonce":810564508},{"type":"v1","addr":"172.21.15.175:6803","nonce":810564508}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6806","nonce":810564508},{"type":"v1","addr":"172.21.15.175:6807","nonce":810564508}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6804","nonce":810564508},{"type":"v1","addr":"172.21.15.175:6805","nonce":810564508}]},"public_addr":"172.21.15.175:6801/810564508","cluster_addr":"172.21.15.175:6803/810564508","heartbeat_back_addr":"172.21.15.175:6807/810564508","heartbeat_front_addr":"172.21.15.175:6805/810564508","state":["exists","up"]},{"osd":5,"uuid":"1c675fbd-f7d6-4535-b420-afaa47e88ad4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6808","nonce":2382596417},{"type":"v1","addr":"172.21.15.175:6809","nonce":2382596417}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6810","nonce":2382596417},{"type":"v1","addr":"172.21.15.175:6811","nonce":2382596417}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6814","nonce":2382596417},{"type":"v1","addr":"172.21.15.175:6815","nonce":2382596417}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6812","nonce":2382596417},{"type":"v1","addr":"172.21.15.175:6813","nonce":2382596417}]},"public_addr":"172.21.15.175:6809/2382596417","cluster_addr":"172.21.15.175:6811/2382596417","heartbeat_back_addr":"172.21.15.175:6815/2382596417","heartbeat_front_addr":"172.21.15.175:6813/2382596417","state":["exists","up"]},{"osd":6,"uuid":"95f278c5-7957-49d9-954a-bbd564ee23f5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":51,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6816","nonce":1936422005},{"type":"v1","addr":"172.21.15.175:6817","nonce":1936422005}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6818","nonce":1936422005},{"type":"v1","addr":"172.21.15.175:6819","nonce":1936422005}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6822","nonce":1936422005},{"type":"v1","addr":"172.21.15.175:6823","nonce":1936422005}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6820","nonce":1936422005},{"type":"v1","addr":"172.21.15.175:6821","nonce":1936422005}]},"public_addr":"172.21.15.175:6817/1936422005","cluster_addr":"172.21.15.175:6819/1936422005","heartbeat_back_addr":"172.21.15.175:6823/1936422005","heartbeat_front_addr":"172.21.15.175:6821/1936422005","state":["exists","up"]},{"osd":7,"uuid":"e24733b0-9e7e-4099-9c13-3d2e9b74a3c2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":56,"up_thru":59,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6824","nonce":1898750451},{"type":"v1","addr":"172.21.15.175:6825","nonce":1898750451}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6826","nonce":1898750451},{"type":"v1","addr":"172.21.15.175:6827","nonce":1898750451}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6830","nonce":1898750451},{"type":"v1","addr":"172.21.15.175:6831","nonce":1898750451}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.175:6828","nonce":1898750451},{"type":"v1","addr":"172.21.15.175:6829","nonce":1898750451}]},"public_addr":"172.21.15.175:6825/1898750451","cluster_addr":"172.21.15.175:6827/1898750451","heartbeat_back_addr":"172.21.15.175:6831/1898750451","heartbeat_front_addr":"172.21.15.175:6829/1898750451","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T10:32:00.399459+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T10:32:32.972827+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T10:33:00.830217+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T10:33:26.137100+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T10:33:48.340351+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T10:34:13.218938+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T10:34:37.659752+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T10:35:01.976363+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.123:0/2332584426":"2024-09-22T10:30:47.165904+0000","172.21.15.123:6801/1080622549":"2024-09-22T10:30:47.165904+0000","172.21.15.123:0/2795289521":"2024-09-22T10:30:47.165904+0000","172.21.15.123:6800/1080622549":"2024-09-22T10:30:47.165904+0000","172.21.15.123:0/1044945966":"2024-09-22T10:30:47.165904+0000","172.21.15.123:6800/1777975386":"2024-09-22T10:29:57.788390+0000","172.21.15.123:0/2899987542":"2024-09-22T10:29:57.788390+0000","172.21.15.123:0/1641759314":"2024-09-22T10:29:57.788390+0000","172.21.15.123:0/1586135573":"2024-09-22T10:29:57.788390+0000","172.21.15.123:0/4085669181":"2024-09-22T10:29:27.633648+0000","172.21.15.123:6801/1777975386":"2024-09-22T10:29:57.788390+0000","172.21.15.123:6800/306380447":"2024-09-22T10:29:27.633648+0000","172.21.15.123:0/3594083655":"2024-09-22T10:29:27.633648+0000","172.21.15.123:6801/306380447":"2024-09-22T10:29:27.633648+0000","172.21.15.123:0/2562221798":"2024-09-22T10:29:27.633648+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T10:35:26.962 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:26 smithi123 ceph-mon[23709]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:35:26.962 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:26 smithi123 ceph-mon[23709]: from='client.? 172.21.15.123:0/2328497459' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T10:35:27.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:26 smithi175 ceph-mon[25806]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:35:27.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:26 smithi175 ceph-mon[25806]: from='client.? 172.21.15.123:0/2328497459' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T10:35:27.413 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -- ceph tell osd.0 flush_pg_stats 2024-09-21T10:35:27.414 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -- ceph tell osd.1 flush_pg_stats 2024-09-21T10:35:27.414 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -- ceph tell osd.2 flush_pg_stats 2024-09-21T10:35:27.415 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -- ceph tell osd.3 flush_pg_stats 2024-09-21T10:35:27.415 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -- ceph tell osd.4 flush_pg_stats 2024-09-21T10:35:27.415 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -- ceph tell osd.5 flush_pg_stats 2024-09-21T10:35:27.416 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -- ceph tell osd.6 flush_pg_stats 2024-09-21T10:35:27.416 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -- ceph tell osd.7 flush_pg_stats 2024-09-21T10:35:27.887 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/mon.smithi123/config 2024-09-21T10:35:27.889 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/mon.smithi123/config 2024-09-21T10:35:27.891 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/mon.smithi123/config 2024-09-21T10:35:27.891 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/mon.smithi123/config 2024-09-21T10:35:27.891 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/mon.smithi123/config 2024-09-21T10:35:27.897 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/mon.smithi123/config 2024-09-21T10:35:27.903 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/mon.smithi123/config 2024-09-21T10:35:27.904 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/mon.smithi123/config 2024-09-21T10:35:28.892 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:28 smithi123 ceph-mon[23709]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:35:29.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:28 smithi175 ceph-mon[25806]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:35:30.735 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:30 smithi123 ceph-mon[23709]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:35:31.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:30 smithi175 ceph-mon[25806]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:35:32.999 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:32 smithi123 ceph-mon[23709]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:35:33.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:32 smithi175 ceph-mon[25806]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:35:34.850 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:34 smithi123 ceph-mon[23709]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:35:35.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:34 smithi175 ceph-mon[25806]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:35:36.682 INFO:teuthology.orchestra.run.smithi123.stdout:240518168585 2024-09-21T10:35:36.682 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.7 2024-09-21T10:35:36.913 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:36 smithi123 ceph-mon[23709]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:35:37.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:36 smithi175 ceph-mon[25806]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:35:37.914 INFO:teuthology.orchestra.run.smithi123.stdout:38654705709 2024-09-21T10:35:37.915 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.0 2024-09-21T10:35:38.725 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:38 smithi123 ceph-mon[23709]: pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:35:38.925 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:38 smithi175 ceph-mon[25806]: pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:35:39.923 INFO:teuthology.orchestra.run.smithi123.stdout:115964117020 2024-09-21T10:35:39.923 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.3 2024-09-21T10:35:40.850 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:40 smithi123 ceph-mon[23709]: pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:35:41.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:40 smithi175 ceph-mon[25806]: pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:35:41.553 INFO:teuthology.orchestra.run.smithi123.stdout:137438953497 2024-09-21T10:35:41.553 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.4 2024-09-21T10:35:42.751 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/mon.smithi123/config 2024-09-21T10:35:42.751 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/mon.smithi123/config 2024-09-21T10:35:43.099 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:42 smithi123 ceph-mon[23709]: pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:35:43.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:42 smithi175 ceph-mon[25806]: pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:35:43.408 INFO:teuthology.orchestra.run.smithi123.stdout:171798691860 2024-09-21T10:35:43.409 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.5 2024-09-21T10:35:44.057 INFO:teuthology.orchestra.run.smithi123.stdout:206158430224 2024-09-21T10:35:44.058 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.6 2024-09-21T10:35:45.070 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:44 smithi123 ceph-mon[23709]: pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:35:45.093 INFO:teuthology.orchestra.run.smithi123.stdout:60129542184 2024-09-21T10:35:45.093 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.1 2024-09-21T10:35:45.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:44 smithi175 ceph-mon[25806]: pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:35:45.301 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/mon.smithi123/config 2024-09-21T10:35:45.705 INFO:teuthology.orchestra.run.smithi123.stdout:81604378659 2024-09-21T10:35:45.706 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.2 2024-09-21T10:35:45.984 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/mon.smithi123/config 2024-09-21T10:35:45.984 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/mon.smithi123/config 2024-09-21T10:35:45.985 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/mon.smithi123/config 2024-09-21T10:35:46.927 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:46 smithi123 ceph-mon[23709]: pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:35:47.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:46 smithi175 ceph-mon[25806]: pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:35:47.936 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:47 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.stnuit/mirror_snapshot_schedule"}]: dispatch 2024-09-21T10:35:47.936 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:47 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.stnuit/trash_purge_schedule"}]: dispatch 2024-09-21T10:35:48.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:47 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.stnuit/mirror_snapshot_schedule"}]: dispatch 2024-09-21T10:35:48.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:47 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.stnuit/trash_purge_schedule"}]: dispatch 2024-09-21T10:35:48.730 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:48 smithi123 ceph-mon[23709]: pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:35:48.731 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:48 smithi123 ceph-mon[23709]: from='client.? 172.21.15.123:0/2339960800' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-21T10:35:48.951 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/mon.smithi123/config 2024-09-21T10:35:49.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:48 smithi175 ceph-mon[25806]: pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:35:49.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:48 smithi175 ceph-mon[25806]: from='client.? 172.21.15.123:0/2339960800' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-21T10:35:49.124 INFO:teuthology.orchestra.run.smithi123.stdout:240518168588 2024-09-21T10:35:49.314 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/mon.smithi123/config 2024-09-21T10:35:50.850 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:50 smithi123 ceph-mon[23709]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:35:51.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:50 smithi175 ceph-mon[25806]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:35:52.707 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:52 smithi123 ceph-mon[23709]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:35:52.731 INFO:tasks.cephadm.ceph_manager.ceph:need seq 240518168585 got 240518168588 for osd.7 2024-09-21T10:35:52.731 DEBUG:teuthology.parallel:result is None 2024-09-21T10:35:53.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:52 smithi175 ceph-mon[25806]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:35:53.796 INFO:teuthology.orchestra.run.smithi123.stdout:115964117022 2024-09-21T10:35:54.392 INFO:tasks.cephadm.ceph_manager.ceph:need seq 115964117020 got 115964117022 for osd.3 2024-09-21T10:35:54.393 DEBUG:teuthology.parallel:result is None 2024-09-21T10:35:54.780 INFO:teuthology.orchestra.run.smithi123.stdout:38654705713 2024-09-21T10:35:55.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:54 smithi123 ceph-mon[23709]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:35:55.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:54 smithi123 ceph-mon[23709]: from='client.? 172.21.15.123:0/2548544421' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-21T10:35:55.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:54 smithi175 ceph-mon[25806]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:35:55.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:54 smithi175 ceph-mon[25806]: from='client.? 172.21.15.123:0/2548544421' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-21T10:35:55.433 INFO:tasks.cephadm.ceph_manager.ceph:need seq 38654705709 got 38654705713 for osd.0 2024-09-21T10:35:55.433 DEBUG:teuthology.parallel:result is None 2024-09-21T10:35:56.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:55 smithi123 ceph-mon[23709]: from='client.? 172.21.15.123:0/4093609423' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-21T10:35:56.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:55 smithi175 ceph-mon[25806]: from='client.? 172.21.15.123:0/4093609423' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-21T10:35:56.394 INFO:teuthology.orchestra.run.smithi123.stdout:206158430227 2024-09-21T10:35:56.963 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:56 smithi123 ceph-mon[23709]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:35:56.963 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:56 smithi123 ceph-mon[23709]: from='client.? 172.21.15.123:0/3756854184' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-21T10:35:57.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:56 smithi175 ceph-mon[25806]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:35:57.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:56 smithi175 ceph-mon[25806]: from='client.? 172.21.15.123:0/3756854184' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-21T10:35:57.219 INFO:tasks.cephadm.ceph_manager.ceph:need seq 206158430224 got 206158430227 for osd.6 2024-09-21T10:35:57.219 DEBUG:teuthology.parallel:result is None 2024-09-21T10:35:58.812 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:58 smithi123 ceph-mon[23709]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:35:58.875 INFO:teuthology.orchestra.run.smithi123.stdout:81604378662 2024-09-21T10:35:59.094 INFO:teuthology.orchestra.run.smithi123.stdout:137438953501 2024-09-21T10:35:59.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:58 smithi175 ceph-mon[25806]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:35:59.498 INFO:tasks.cephadm.ceph_manager.ceph:need seq 81604378659 got 81604378662 for osd.2 2024-09-21T10:35:59.498 DEBUG:teuthology.parallel:result is None 2024-09-21T10:35:59.817 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:59 smithi123 ceph-mon[23709]: from='client.? 172.21.15.123:0/2794680380' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-21T10:35:59.817 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:35:59 smithi123 ceph-mon[23709]: from='client.? 172.21.15.123:0/900884198' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-21T10:36:00.103 INFO:teuthology.orchestra.run.smithi123.stdout:60129542187 2024-09-21T10:36:00.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:59 smithi175 ceph-mon[25806]: from='client.? 172.21.15.123:0/2794680380' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-21T10:36:00.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:35:59 smithi175 ceph-mon[25806]: from='client.? 172.21.15.123:0/900884198' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-21T10:36:00.332 INFO:teuthology.orchestra.run.smithi123.stdout:171798691863 2024-09-21T10:36:00.838 INFO:tasks.cephadm.ceph_manager.ceph:need seq 60129542184 got 60129542187 for osd.1 2024-09-21T10:36:00.838 DEBUG:teuthology.parallel:result is None 2024-09-21T10:36:00.962 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:00 smithi123 ceph-mon[23709]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:36:00.962 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:00 smithi123 ceph-mon[23709]: from='client.? 172.21.15.123:0/1256388284' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-21T10:36:00.963 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:00 smithi123 ceph-mon[23709]: from='client.? 172.21.15.123:0/3823509808' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-21T10:36:01.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:00 smithi175 ceph-mon[25806]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:36:01.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:00 smithi175 ceph-mon[25806]: from='client.? 172.21.15.123:0/1256388284' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-21T10:36:01.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:00 smithi175 ceph-mon[25806]: from='client.? 172.21.15.123:0/3823509808' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-21T10:36:01.449 INFO:tasks.cephadm.ceph_manager.ceph:need seq 171798691860 got 171798691863 for osd.5 2024-09-21T10:36:01.449 DEBUG:teuthology.parallel:result is None 2024-09-21T10:36:01.768 INFO:tasks.cephadm.ceph_manager.ceph:need seq 137438953497 got 137438953501 for osd.4 2024-09-21T10:36:01.769 DEBUG:teuthology.parallel:result is None 2024-09-21T10:36:01.769 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-09-21T10:36:01.769 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T10:36:01.972 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/mon.smithi123/config 2024-09-21T10:36:03.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:02 smithi123 ceph-mon[23709]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:36:03.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:02 smithi175 ceph-mon[25806]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:36:03.756 INFO:teuthology.orchestra.run.smithi123.stdout: 2024-09-21T10:36:03.758 INFO:teuthology.orchestra.run.smithi123.stderr:dumped all 2024-09-21T10:36:04.391 INFO:teuthology.orchestra.run.smithi123.stdout:{"pg_ready":true,"pg_map":{"version":206,"stamp":"2024-09-21T10:36:03.625330+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48868,"kb_used_data":6372,"kb_used_omap":0,"kb_used_meta":42432,"kb_avail":749912348,"statfs":{"total":767960285184,"available":767910244352,"internally_reserved":0,"allocated":6524928,"data_stored":3571463,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43450368},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003019"},"pg_stats":[{"pgid":"1.0","version":"23'117","reported_seq":226,"reported_epoch":60,"state":"active+clean","last_fresh":"2024-09-21T10:35:07.888225+0000","last_change":"2024-09-21T10:35:07.888225+0000","last_active":"2024-09-21T10:35:07.888225+0000","last_peered":"2024-09-21T10:35:07.888225+0000","last_clean":"2024-09-21T10:35:07.888225+0000","last_became_active":"2024-09-21T10:35:07.887820+0000","last_became_peered":"2024-09-21T10:35:07.887820+0000","last_unstale":"2024-09-21T10:35:07.888225+0000","last_undegraded":"2024-09-21T10:35:07.888225+0000","last_fullsized":"2024-09-21T10:35:07.888225+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":21,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:33:04.516401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:33:04.516401+0000","last_clean_scrub_stamp":"2024-09-21T10:33:04.516401+0000","objects_scrubbed":0,"log_size":117,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:19:53.776233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3727360,"data_stored":3702496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":56,"seq":240518168591,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6056,"kb_used_data":1120,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739096,"statfs":{"total":95995035648,"available":95988834304,"internally_reserved":0,"allocated":1146880,"data_stored":776872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.625}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.70199999999999996}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.71499999999999997}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.752}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56299999999999994}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57699999999999996}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.60299999999999998}]}]},{"osd":6,"up_from":48,"seq":206158430228,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6116,"kb_used_data":1116,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739036,"statfs":{"total":95995035648,"available":95988772864,"internally_reserved":0,"allocated":1142784,"data_stored":776657,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 10:35:40 2024","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.57299999999999995}]},{"osd":1,"last update":"Sat Sep 21 10:35:40 2024","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.754,"5min":0.754,"15min":0.754},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":0.79700000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.56599999999999995}]},{"osd":2,"last update":"Sat Sep 21 10:35:40 2024","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.60799999999999998},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.495}]},{"osd":3,"last update":"Sat Sep 21 10:35:40 2024","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.53900000000000003}]},{"osd":4,"last update":"Sat Sep 21 10:35:40 2024","interfaces":[{"interface":"back","average":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.63300000000000001},{"interface":"front","average":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.64800000000000002}]},{"osd":5,"last update":"Sat Sep 21 10:35:40 2024","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":0.499,"15min":0.499},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.60099999999999998},{"interface":"front","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"last":0.66800000000000004}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.72999999999999998}]}]},{"osd":1,"up_from":14,"seq":60129542188,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6372,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738780,"statfs":{"total":95995035648,"available":95988510720,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 10:35:43 2024","interfaces":[{"interface":"back","average":{"1min":0.374,"5min":0.34100000000000003,"15min":0.32000000000000001},"min":{"1min":0.23999999999999999,"5min":0.215,"15min":0.215},"max":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.36899999999999999,"5min":0.35199999999999998,"15min":0.34100000000000003},"min":{"1min":0.26400000000000001,"5min":0.221,"15min":0.221},"max":{"1min":0.56399999999999995,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.63800000000000001}]},{"osd":2,"last update":"Sat Sep 21 10:35:06 2024","interfaces":[{"interface":"back","average":{"1min":0.40699999999999997,"5min":0.38700000000000001,"15min":0.38400000000000001},"min":{"1min":0.27900000000000003,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.52300000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"last":0.56000000000000005},{"interface":"front","average":{"1min":0.40600000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"min":{"1min":0.26700000000000002,"5min":0.249,"15min":0.249},"max":{"1min":0.60199999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.54200000000000004}]},{"osd":3,"last update":"Sat Sep 21 10:35:32 2024","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"min":{"1min":0.311,"5min":0.245,"15min":0.245},"max":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.45500000000000002,"5min":0.44700000000000001,"15min":0.44500000000000001},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.65100000000000002}]},{"osd":4,"last update":"Sat Sep 21 10:35:53 2024","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":0.57799999999999996,"15min":0.57099999999999995},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.89100000000000001,"5min":0.91000000000000003,"15min":0.91000000000000003},"last":0.98699999999999999},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.57599999999999996,"15min":0.56899999999999995},"min":{"1min":0.48299999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.95199999999999996}]},{"osd":5,"last update":"Sat Sep 21 10:35:20 2024","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.88500000000000001},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.84799999999999998}]},{"osd":6,"last update":"Sat Sep 21 10:35:38 2024","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.85999999999999999},{"interface":"front","average":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"min":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":0.94199999999999995,"5min":0.94199999999999995,"15min":0.94199999999999995},"last":0.96199999999999997}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.92100000000000004}]}]},{"osd":0,"up_from":9,"seq":38654705715,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6892,"kb_used_data":1124,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738260,"statfs":{"total":95995035648,"available":95987978240,"internally_reserved":0,"allocated":1150976,"data_stored":777195,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 21 10:35:38 2024","interfaces":[{"interface":"back","average":{"1min":0.378,"5min":0.33100000000000002,"15min":0.308},"min":{"1min":0.23300000000000001,"5min":0.217,"15min":0.217},"max":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"last":0.33200000000000002},{"interface":"front","average":{"1min":0.40000000000000002,"5min":0.34599999999999997,"15min":0.32000000000000001},"min":{"1min":0.28799999999999998,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.53500000000000003,"5min":0.57499999999999996,"15min":0.57499999999999996},"last":0.39300000000000002}]},{"osd":2,"last update":"Sat Sep 21 10:35:05 2024","interfaces":[{"interface":"back","average":{"1min":0.41399999999999998,"5min":0.40500000000000003,"15min":0.40300000000000002},"min":{"1min":0.28000000000000003,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"last":0.27900000000000003},{"interface":"front","average":{"1min":0.40600000000000003,"5min":0.38700000000000001,"15min":0.38400000000000001},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"last":0.30499999999999999}]},{"osd":3,"last update":"Sat Sep 21 10:35:31 2024","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":0.44,"15min":0.439},"min":{"1min":0.317,"5min":0.307,"15min":0.307},"max":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.42199999999999999,"5min":0.42699999999999999,"15min":0.42799999999999999},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.52400000000000002,"5min":0.54200000000000004,"15min":0.54200000000000004},"last":0.29299999999999998}]},{"osd":4,"last update":"Sat Sep 21 10:35:54 2024","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.60899999999999999,"15min":0.61099999999999999},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.89500000000000002,"5min":1.4339999999999999,"15min":1.4339999999999999},"last":0.60399999999999998},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.61399999999999999,"15min":0.61799999999999999},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.74399999999999999,"5min":1.4079999999999999,"15min":1.4079999999999999},"last":0.495}]},{"osd":5,"last update":"Sat Sep 21 10:35:16 2024","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.94999999999999996,"5min":0.94999999999999996,"15min":0.94999999999999996},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.88800000000000001,"5min":0.88800000000000001,"15min":0.88800000000000001},"last":0.54800000000000004}]},{"osd":6,"last update":"Sat Sep 21 10:35:39 2024","interfaces":[{"interface":"back","average":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":1.379,"5min":1.379,"15min":1.379},"last":0.55700000000000005},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"min":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":0.85699999999999998,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.46400000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58299999999999996}]}]},{"osd":2,"up_from":19,"seq":81604378664,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6244,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5632,"kb_avail":93738908,"statfs":{"total":95995035648,"available":95988641792,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5767168},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 10:35:04 2024","interfaces":[{"interface":"back","average":{"1min":0.40300000000000002,"5min":0.35599999999999998,"15min":0.34799999999999998},"min":{"1min":0.26100000000000001,"5min":0.247,"15min":0.247},"max":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"last":0.30599999999999999},{"interface":"front","average":{"1min":0.436,"5min":0.35399999999999998,"15min":0.34000000000000002},"min":{"1min":0.27600000000000002,"5min":0.187,"15min":0.187},"max":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"last":0.23499999999999999}]},{"osd":1,"last update":"Sat Sep 21 10:35:04 2024","interfaces":[{"interface":"back","average":{"1min":0.45100000000000001,"5min":0.38100000000000001,"15min":0.36899999999999999},"min":{"1min":0.28599999999999998,"5min":0.216,"15min":0.216},"max":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"last":0.33700000000000002},{"interface":"front","average":{"1min":0.438,"5min":0.37,"15min":0.35899999999999999},"min":{"1min":0.32200000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"last":0.218}]},{"osd":3,"last update":"Sat Sep 21 10:35:30 2024","interfaces":[{"interface":"back","average":{"1min":0.46700000000000003,"5min":0.439,"15min":0.435},"min":{"1min":0.32700000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.64700000000000002,"5min":0.67000000000000004,"15min":0.67000000000000004},"last":0.25},{"interface":"front","average":{"1min":0.45000000000000001,"5min":0.42099999999999999,"15min":0.41599999999999998},"min":{"1min":0.36799999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"last":0.28999999999999998}]},{"osd":4,"last update":"Sat Sep 21 10:35:52 2024","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"min":{"1min":0.45100000000000001,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.94799999999999995,"5min":1.4830000000000001,"15min":1.4830000000000001},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.61099999999999999,"15min":0.60599999999999998},"min":{"1min":0.44400000000000001,"5min":0.434,"15min":0.434},"max":{"1min":1.0089999999999999,"5min":1.0129999999999999,"15min":1.0129999999999999},"last":0.56699999999999995}]},{"osd":5,"last update":"Sat Sep 21 10:35:18 2024","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.627,"15min":0.627},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":1.1559999999999999,"5min":1.1559999999999999,"15min":1.1559999999999999},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"min":{"1min":0.505,"5min":0.505,"15min":0.505},"max":{"1min":1.248,"5min":1.248,"15min":1.248},"last":0.50900000000000001}]},{"osd":6,"last update":"Sat Sep 21 10:35:40 2024","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":1.0740000000000001,"5min":1.0740000000000001,"15min":1.0740000000000001},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":1.1870000000000001,"5min":1.1870000000000001,"15min":1.1870000000000001},"last":0.61199999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.59499999999999997}]}]},{"osd":3,"up_from":27,"seq":115964117024,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5732,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739420,"statfs":{"total":95995035648,"available":95989166080,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 10:35:30 2024","interfaces":[{"interface":"back","average":{"1min":0.41699999999999998,"5min":0.39000000000000001,"15min":0.38600000000000001},"min":{"1min":0.29199999999999998,"5min":0.249,"15min":0.249},"max":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"last":0.30499999999999999},{"interface":"front","average":{"1min":0.41099999999999998,"5min":0.41699999999999998,"15min":0.41799999999999998},"min":{"1min":0.27600000000000002,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.39100000000000001}]},{"osd":1,"last update":"Sat Sep 21 10:35:30 2024","interfaces":[{"interface":"back","average":{"1min":0.434,"5min":0.42299999999999999,"15min":0.42099999999999999},"min":{"1min":0.29999999999999999,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.58699999999999997,"5min":0.65300000000000002,"15min":0.65300000000000002},"last":0.28699999999999998},{"interface":"front","average":{"1min":0.42899999999999999,"5min":0.442,"15min":0.44400000000000001},"min":{"1min":0.29199999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.67100000000000004,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.45300000000000001}]},{"osd":2,"last update":"Sat Sep 21 10:35:30 2024","interfaces":[{"interface":"back","average":{"1min":0.436,"5min":0.44400000000000001,"15min":0.44500000000000001},"min":{"1min":0.316,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.435,"5min":0.44,"15min":0.441},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"last":0.35199999999999998}]},{"osd":4,"last update":"Sat Sep 21 10:35:54 2024","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.59299999999999997,"15min":0.60299999999999998},"min":{"1min":0.437,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.746,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.82199999999999995},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.63900000000000001,"15min":0.64700000000000002},"min":{"1min":0.44900000000000001,"5min":0.437,"15min":0.437},"max":{"1min":0.84899999999999998,"5min":0.94999999999999996,"15min":0.94999999999999996},"last":0.76900000000000002}]},{"osd":5,"last update":"Sat Sep 21 10:35:17 2024","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.92200000000000004,"5min":0.92200000000000004,"15min":0.92200000000000004},"last":0.70499999999999996},{"interface":"front","average":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.93700000000000006,"5min":0.93700000000000006,"15min":0.93700000000000006},"last":0.75800000000000001}]},{"osd":6,"last update":"Sat Sep 21 10:35:44 2024","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":0.62,"15min":0.62},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.88100000000000001,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.74099999999999999},{"interface":"front","average":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"min":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":1.095,"5min":1.095,"15min":1.095},"last":0.78000000000000003}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.69899999999999995}]}]},{"osd":4,"up_from":32,"seq":137438953502,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5792,"kb_used_data":600,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739360,"statfs":{"total":95995035648,"available":95989104640,"internally_reserved":0,"allocated":614400,"data_stored":247994,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 10:35:54 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.55400000000000005,"15min":0.55700000000000005},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.69899999999999995,"5min":0.97899999999999998,"15min":0.97899999999999998},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.58299999999999996,"15min":0.58699999999999997},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.70699999999999996,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.46700000000000003}]},{"osd":1,"last update":"Sat Sep 21 10:35:54 2024","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.55700000000000005,"15min":0.55600000000000005},"min":{"1min":0.42499999999999999,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.69899999999999995,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.372},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.54500000000000004,"15min":0.54600000000000004},"min":{"1min":0.39200000000000002,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.67900000000000005,"5min":0.92200000000000004,"15min":0.92200000000000004},"last":0.498}]},{"osd":2,"last update":"Sat Sep 21 10:35:54 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.56499999999999995,"15min":0.56799999999999995},"min":{"1min":0.38,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.66600000000000004,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.40999999999999998},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.55100000000000005,"15min":0.55500000000000005},"min":{"1min":0.42799999999999999,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.65400000000000003,"5min":0.747,"15min":0.747},"last":0.52000000000000002}]},{"osd":3,"last update":"Sat Sep 21 10:35:54 2024","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.56899999999999995,"15min":0.56999999999999995},"min":{"1min":0.38900000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.72399999999999998,"5min":0.88200000000000001,"15min":0.88200000000000001},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.55400000000000005,"15min":0.54900000000000004},"min":{"1min":0.46700000000000003,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.73999999999999999,"5min":0.94199999999999995,"15min":0.94199999999999995},"last":0.42899999999999999}]},{"osd":5,"last update":"Sat Sep 21 10:35:15 2024","interfaces":[{"interface":"back","average":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.624,"5min":0.624,"15min":0.624},"last":0.58399999999999996}]},{"osd":6,"last update":"Sat Sep 21 10:35:42 2024","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.48699999999999999,"15min":0.48699999999999999},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.59699999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57899999999999996}]}]},{"osd":5,"up_from":40,"seq":171798691864,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5664,"kb_used_data":600,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739488,"statfs":{"total":95995035648,"available":95989235712,"internally_reserved":0,"allocated":614400,"data_stored":247944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 10:35:20 2024","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.45000000000000001},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.56899999999999995}]},{"osd":1,"last update":"Sat Sep 21 10:35:20 2024","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.51900000000000002}]},{"osd":2,"last update":"Sat Sep 21 10:35:20 2024","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.59399999999999997},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.498}]},{"osd":3,"last update":"Sat Sep 21 10:35:20 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.58299999999999996}]},{"osd":4,"last update":"Sat Sep 21 10:35:20 2024","interfaces":[{"interface":"back","average":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"last":0.61599999999999999},{"interface":"front","average":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"last":0.63500000000000001}]},{"osd":6,"last update":"Sat Sep 21 10:35:40 2024","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"last":0.65900000000000003},{"interface":"front","average":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"last":0.624}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.64800000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T10:36:04.393 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T10:36:04.597 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/mon.smithi123/config 2024-09-21T10:36:04.965 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:04 smithi123 ceph-mon[23709]: pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:36:04.965 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:04 smithi123 ceph-mon[23709]: from='client.14452 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:36:05.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:04 smithi175 ceph-mon[25806]: pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:36:05.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:04 smithi175 ceph-mon[25806]: from='client.14452 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:36:06.300 INFO:teuthology.orchestra.run.smithi123.stdout: 2024-09-21T10:36:06.302 INFO:teuthology.orchestra.run.smithi123.stderr:dumped all 2024-09-21T10:36:06.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:06 smithi123 ceph-mon[23709]: pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:36:06.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:06 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:36:06.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:06 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:36:06.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:06 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T10:36:06.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:06 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:36:06.979 INFO:teuthology.orchestra.run.smithi123.stdout:{"pg_ready":true,"pg_map":{"version":207,"stamp":"2024-09-21T10:36:05.625600+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48868,"kb_used_data":6372,"kb_used_omap":0,"kb_used_meta":42432,"kb_avail":749912348,"statfs":{"total":767960285184,"available":767910244352,"internally_reserved":0,"allocated":6524928,"data_stored":3571463,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43450368},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001874"},"pg_stats":[{"pgid":"1.0","version":"23'117","reported_seq":226,"reported_epoch":60,"state":"active+clean","last_fresh":"2024-09-21T10:35:07.888225+0000","last_change":"2024-09-21T10:35:07.888225+0000","last_active":"2024-09-21T10:35:07.888225+0000","last_peered":"2024-09-21T10:35:07.888225+0000","last_clean":"2024-09-21T10:35:07.888225+0000","last_became_active":"2024-09-21T10:35:07.887820+0000","last_became_peered":"2024-09-21T10:35:07.887820+0000","last_unstale":"2024-09-21T10:35:07.888225+0000","last_undegraded":"2024-09-21T10:35:07.888225+0000","last_fullsized":"2024-09-21T10:35:07.888225+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":21,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T10:33:04.516401+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T10:33:04.516401+0000","last_clean_scrub_stamp":"2024-09-21T10:33:04.516401+0000","objects_scrubbed":0,"log_size":117,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:19:53.776233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3727360,"data_stored":3702496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":56,"seq":240518168592,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6056,"kb_used_data":1120,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739096,"statfs":{"total":95995035648,"available":95988834304,"internally_reserved":0,"allocated":1146880,"data_stored":776872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 10:36:04 2024","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.65600000000000003},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.66300000000000003}]},{"osd":1,"last update":"Sat Sep 21 10:36:04 2024","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.84999999999999998,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.71499999999999997}]},{"osd":2,"last update":"Sat Sep 21 10:36:04 2024","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"min":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.68000000000000005},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"min":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":0.91200000000000003,"5min":0.91200000000000003,"15min":0.91200000000000003},"last":0.73599999999999999}]},{"osd":3,"last update":"Sat Sep 21 10:36:04 2024","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":0.85799999999999998,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.64700000000000002},{"interface":"front","average":{"1min":0.624,"5min":0.624,"15min":0.624},"min":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"max":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.69199999999999995}]},{"osd":4,"last update":"Sat Sep 21 10:36:04 2024","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.503},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.52200000000000002}]},{"osd":5,"last update":"Sat Sep 21 10:36:04 2024","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":1.0349999999999999,"5min":1.0349999999999999,"15min":1.0349999999999999},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.54700000000000004}]},{"osd":6,"last update":"Sat Sep 21 10:36:04 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.628},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.57499999999999996}]}]},{"osd":6,"up_from":48,"seq":206158430229,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6116,"kb_used_data":1116,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739036,"statfs":{"total":95995035648,"available":95988772864,"internally_reserved":0,"allocated":1142784,"data_stored":776657,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 10:35:40 2024","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.36099999999999999},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.57899999999999996}]},{"osd":1,"last update":"Sat Sep 21 10:35:40 2024","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.754,"5min":0.754,"15min":0.754},"last":0.44700000000000001},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":0.79700000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.55200000000000005}]},{"osd":2,"last update":"Sat Sep 21 10:35:40 2024","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.39100000000000001}]},{"osd":3,"last update":"Sat Sep 21 10:35:40 2024","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.496}]},{"osd":4,"last update":"Sat Sep 21 10:35:40 2024","interfaces":[{"interface":"back","average":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.51400000000000001}]},{"osd":5,"last update":"Sat Sep 21 10:35:40 2024","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":0.499,"15min":0.499},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.628},{"interface":"front","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"last":0.52300000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.64100000000000001}]}]},{"osd":1,"up_from":14,"seq":60129542189,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6372,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738780,"statfs":{"total":95995035648,"available":95988510720,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 10:35:43 2024","interfaces":[{"interface":"back","average":{"1min":0.374,"5min":0.34100000000000003,"15min":0.32000000000000001},"min":{"1min":0.23999999999999999,"5min":0.215,"15min":0.215},"max":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.36899999999999999,"5min":0.35199999999999998,"15min":0.34100000000000003},"min":{"1min":0.26400000000000001,"5min":0.221,"15min":0.221},"max":{"1min":0.56399999999999995,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.34799999999999998}]},{"osd":2,"last update":"Sat Sep 21 10:35:06 2024","interfaces":[{"interface":"back","average":{"1min":0.40699999999999997,"5min":0.38700000000000001,"15min":0.38400000000000001},"min":{"1min":0.27900000000000003,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.52300000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"last":0.33600000000000002},{"interface":"front","average":{"1min":0.40600000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"min":{"1min":0.26700000000000002,"5min":0.249,"15min":0.249},"max":{"1min":0.60199999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.26900000000000002}]},{"osd":3,"last update":"Sat Sep 21 10:35:32 2024","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"min":{"1min":0.311,"5min":0.245,"15min":0.245},"max":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"last":0.32100000000000001},{"interface":"front","average":{"1min":0.45500000000000002,"5min":0.44700000000000001,"15min":0.44500000000000001},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.30399999999999999}]},{"osd":4,"last update":"Sat Sep 21 10:35:53 2024","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":0.57799999999999996,"15min":0.57099999999999995},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.89100000000000001,"5min":0.91000000000000003,"15min":0.91000000000000003},"last":0.55500000000000005},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.57599999999999996,"15min":0.56899999999999995},"min":{"1min":0.48299999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.65900000000000003}]},{"osd":5,"last update":"Sat Sep 21 10:35:20 2024","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.64900000000000002},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.54200000000000004}]},{"osd":6,"last update":"Sat Sep 21 10:35:38 2024","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.60799999999999998},{"interface":"front","average":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"min":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":0.94199999999999995,"5min":0.94199999999999995,"15min":0.94199999999999995},"last":0.67300000000000004}]},{"osd":7,"last update":"Sat Sep 21 10:36:04 2024","interfaces":[{"interface":"back","average":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"min":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"max":{"1min":0.92100000000000004,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":0.625},{"interface":"front","average":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.97399999999999998,"5min":0.97399999999999998,"15min":0.97399999999999998},"last":0.56899999999999995}]}]},{"osd":0,"up_from":9,"seq":38654705715,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6892,"kb_used_data":1124,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738260,"statfs":{"total":95995035648,"available":95987978240,"internally_reserved":0,"allocated":1150976,"data_stored":777195,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 21 10:35:38 2024","interfaces":[{"interface":"back","average":{"1min":0.378,"5min":0.33100000000000002,"15min":0.308},"min":{"1min":0.23300000000000001,"5min":0.217,"15min":0.217},"max":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"last":0.33200000000000002},{"interface":"front","average":{"1min":0.40000000000000002,"5min":0.34599999999999997,"15min":0.32000000000000001},"min":{"1min":0.28799999999999998,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.53500000000000003,"5min":0.57499999999999996,"15min":0.57499999999999996},"last":0.39300000000000002}]},{"osd":2,"last update":"Sat Sep 21 10:35:05 2024","interfaces":[{"interface":"back","average":{"1min":0.41399999999999998,"5min":0.40500000000000003,"15min":0.40300000000000002},"min":{"1min":0.28000000000000003,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"last":0.27900000000000003},{"interface":"front","average":{"1min":0.40600000000000003,"5min":0.38700000000000001,"15min":0.38400000000000001},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"last":0.30499999999999999}]},{"osd":3,"last update":"Sat Sep 21 10:35:31 2024","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":0.44,"15min":0.439},"min":{"1min":0.317,"5min":0.307,"15min":0.307},"max":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.42199999999999999,"5min":0.42699999999999999,"15min":0.42799999999999999},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.52400000000000002,"5min":0.54200000000000004,"15min":0.54200000000000004},"last":0.29299999999999998}]},{"osd":4,"last update":"Sat Sep 21 10:35:54 2024","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.60899999999999999,"15min":0.61099999999999999},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.89500000000000002,"5min":1.4339999999999999,"15min":1.4339999999999999},"last":0.60399999999999998},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.61399999999999999,"15min":0.61799999999999999},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.74399999999999999,"5min":1.4079999999999999,"15min":1.4079999999999999},"last":0.495}]},{"osd":5,"last update":"Sat Sep 21 10:35:16 2024","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.94999999999999996,"5min":0.94999999999999996,"15min":0.94999999999999996},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.88800000000000001,"5min":0.88800000000000001,"15min":0.88800000000000001},"last":0.54800000000000004}]},{"osd":6,"last update":"Sat Sep 21 10:35:39 2024","interfaces":[{"interface":"back","average":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":1.379,"5min":1.379,"15min":1.379},"last":0.55700000000000005},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"min":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":0.85699999999999998,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.46400000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58299999999999996}]}]},{"osd":2,"up_from":19,"seq":81604378664,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6244,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5632,"kb_avail":93738908,"statfs":{"total":95995035648,"available":95988641792,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5767168},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 10:35:04 2024","interfaces":[{"interface":"back","average":{"1min":0.40300000000000002,"5min":0.35599999999999998,"15min":0.34799999999999998},"min":{"1min":0.26100000000000001,"5min":0.247,"15min":0.247},"max":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"last":0.30599999999999999},{"interface":"front","average":{"1min":0.436,"5min":0.35399999999999998,"15min":0.34000000000000002},"min":{"1min":0.27600000000000002,"5min":0.187,"15min":0.187},"max":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"last":0.23499999999999999}]},{"osd":1,"last update":"Sat Sep 21 10:35:04 2024","interfaces":[{"interface":"back","average":{"1min":0.45100000000000001,"5min":0.38100000000000001,"15min":0.36899999999999999},"min":{"1min":0.28599999999999998,"5min":0.216,"15min":0.216},"max":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"last":0.33700000000000002},{"interface":"front","average":{"1min":0.438,"5min":0.37,"15min":0.35899999999999999},"min":{"1min":0.32200000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"last":0.218}]},{"osd":3,"last update":"Sat Sep 21 10:35:30 2024","interfaces":[{"interface":"back","average":{"1min":0.46700000000000003,"5min":0.439,"15min":0.435},"min":{"1min":0.32700000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.64700000000000002,"5min":0.67000000000000004,"15min":0.67000000000000004},"last":0.25},{"interface":"front","average":{"1min":0.45000000000000001,"5min":0.42099999999999999,"15min":0.41599999999999998},"min":{"1min":0.36799999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"last":0.28999999999999998}]},{"osd":4,"last update":"Sat Sep 21 10:35:52 2024","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"min":{"1min":0.45100000000000001,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.94799999999999995,"5min":1.4830000000000001,"15min":1.4830000000000001},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.61099999999999999,"15min":0.60599999999999998},"min":{"1min":0.44400000000000001,"5min":0.434,"15min":0.434},"max":{"1min":1.0089999999999999,"5min":1.0129999999999999,"15min":1.0129999999999999},"last":0.56699999999999995}]},{"osd":5,"last update":"Sat Sep 21 10:35:18 2024","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.627,"15min":0.627},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":1.1559999999999999,"5min":1.1559999999999999,"15min":1.1559999999999999},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"min":{"1min":0.505,"5min":0.505,"15min":0.505},"max":{"1min":1.248,"5min":1.248,"15min":1.248},"last":0.50900000000000001}]},{"osd":6,"last update":"Sat Sep 21 10:35:40 2024","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":1.0740000000000001,"5min":1.0740000000000001,"15min":1.0740000000000001},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":1.1870000000000001,"5min":1.1870000000000001,"15min":1.1870000000000001},"last":0.61199999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.59499999999999997}]}]},{"osd":3,"up_from":27,"seq":115964117025,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5732,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739420,"statfs":{"total":95995035648,"available":95989166080,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 10:35:30 2024","interfaces":[{"interface":"back","average":{"1min":0.41699999999999998,"5min":0.39000000000000001,"15min":0.38600000000000001},"min":{"1min":0.29199999999999998,"5min":0.249,"15min":0.249},"max":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"last":0.26100000000000001},{"interface":"front","average":{"1min":0.41099999999999998,"5min":0.41699999999999998,"15min":0.41799999999999998},"min":{"1min":0.27600000000000002,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.34100000000000003}]},{"osd":1,"last update":"Sat Sep 21 10:35:30 2024","interfaces":[{"interface":"back","average":{"1min":0.434,"5min":0.42299999999999999,"15min":0.42099999999999999},"min":{"1min":0.29999999999999999,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.58699999999999997,"5min":0.65300000000000002,"15min":0.65300000000000002},"last":0.30199999999999999},{"interface":"front","average":{"1min":0.42899999999999999,"5min":0.442,"15min":0.44400000000000001},"min":{"1min":0.29199999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.67100000000000004,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.314}]},{"osd":2,"last update":"Sat Sep 21 10:35:30 2024","interfaces":[{"interface":"back","average":{"1min":0.436,"5min":0.44400000000000001,"15min":0.44500000000000001},"min":{"1min":0.316,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"last":0.35599999999999998},{"interface":"front","average":{"1min":0.435,"5min":0.44,"15min":0.441},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"last":0.32900000000000001}]},{"osd":4,"last update":"Sat Sep 21 10:35:54 2024","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.59299999999999997,"15min":0.60299999999999998},"min":{"1min":0.437,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.746,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.66800000000000004},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.63900000000000001,"15min":0.64700000000000002},"min":{"1min":0.44900000000000001,"5min":0.437,"15min":0.437},"max":{"1min":0.84899999999999998,"5min":0.94999999999999996,"15min":0.94999999999999996},"last":0.61299999999999999}]},{"osd":5,"last update":"Sat Sep 21 10:35:17 2024","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.92200000000000004,"5min":0.92200000000000004,"15min":0.92200000000000004},"last":0.64800000000000002},{"interface":"front","average":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.93700000000000006,"5min":0.93700000000000006,"15min":0.93700000000000006},"last":0.62}]},{"osd":6,"last update":"Sat Sep 21 10:35:44 2024","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":0.62,"15min":0.62},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.88100000000000001,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.68600000000000005},{"interface":"front","average":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"min":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":1.095,"5min":1.095,"15min":1.095},"last":0.59299999999999997}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.70999999999999996}]}]},{"osd":4,"up_from":32,"seq":137438953502,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5792,"kb_used_data":600,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739360,"statfs":{"total":95995035648,"available":95989104640,"internally_reserved":0,"allocated":614400,"data_stored":247994,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 10:35:54 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.55400000000000005,"15min":0.55700000000000005},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.69899999999999995,"5min":0.97899999999999998,"15min":0.97899999999999998},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.58299999999999996,"15min":0.58699999999999997},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.70699999999999996,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.46700000000000003}]},{"osd":1,"last update":"Sat Sep 21 10:35:54 2024","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.55700000000000005,"15min":0.55600000000000005},"min":{"1min":0.42499999999999999,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.69899999999999995,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.372},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.54500000000000004,"15min":0.54600000000000004},"min":{"1min":0.39200000000000002,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.67900000000000005,"5min":0.92200000000000004,"15min":0.92200000000000004},"last":0.498}]},{"osd":2,"last update":"Sat Sep 21 10:35:54 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.56499999999999995,"15min":0.56799999999999995},"min":{"1min":0.38,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.66600000000000004,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.40999999999999998},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.55100000000000005,"15min":0.55500000000000005},"min":{"1min":0.42799999999999999,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.65400000000000003,"5min":0.747,"15min":0.747},"last":0.52000000000000002}]},{"osd":3,"last update":"Sat Sep 21 10:35:54 2024","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.56899999999999995,"15min":0.56999999999999995},"min":{"1min":0.38900000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.72399999999999998,"5min":0.88200000000000001,"15min":0.88200000000000001},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.55400000000000005,"15min":0.54900000000000004},"min":{"1min":0.46700000000000003,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.73999999999999999,"5min":0.94199999999999995,"15min":0.94199999999999995},"last":0.42899999999999999}]},{"osd":5,"last update":"Sat Sep 21 10:35:15 2024","interfaces":[{"interface":"back","average":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.624,"5min":0.624,"15min":0.624},"last":0.58399999999999996}]},{"osd":6,"last update":"Sat Sep 21 10:35:42 2024","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.48699999999999999,"15min":0.48699999999999999},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.59699999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57899999999999996}]}]},{"osd":5,"up_from":40,"seq":171798691865,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5664,"kb_used_data":600,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739488,"statfs":{"total":95995035648,"available":95989235712,"internally_reserved":0,"allocated":614400,"data_stored":247944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 10:35:20 2024","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.48299999999999998}]},{"osd":1,"last update":"Sat Sep 21 10:35:20 2024","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.61399999999999999},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.66900000000000004}]},{"osd":2,"last update":"Sat Sep 21 10:35:20 2024","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.57599999999999996}]},{"osd":3,"last update":"Sat Sep 21 10:35:20 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.65400000000000003}]},{"osd":4,"last update":"Sat Sep 21 10:35:20 2024","interfaces":[{"interface":"back","average":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"last":0.71699999999999997},{"interface":"front","average":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"last":0.54900000000000004}]},{"osd":6,"last update":"Sat Sep 21 10:35:40 2024","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"last":0.69899999999999995}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.73399999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T10:36:06.980 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-09-21T10:36:06.980 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-09-21T10:36:06.980 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-09-21T10:36:06.981 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -- ceph health --format=json 2024-09-21T10:36:07.126 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:06 smithi175 ceph-mon[25806]: pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:36:07.126 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:06 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:36:07.126 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:06 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:36:07.126 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:06 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T10:36:07.126 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:06 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:36:07.191 INFO:teuthology.orchestra.run.smithi123.stderr:Inferring config /var/lib/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/mon.smithi123/config 2024-09-21T10:36:07.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:07 smithi123 ceph-mon[23709]: from='client.14456 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:36:08.025 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:07 smithi175 ceph-mon[25806]: from='client.14456 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:36:08.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:08 smithi123 ceph-mon[23709]: pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:36:08.869 INFO:teuthology.orchestra.run.smithi123.stdout: 2024-09-21T10:36:08.869 INFO:teuthology.orchestra.run.smithi123.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-09-21T10:36:09.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:08 smithi175 ceph-mon[25806]: pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:36:09.477 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-09-21T10:36:09.477 INFO:tasks.cephadm:Setup complete, yielding 2024-09-21T10:36:09.477 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-21T10:36:09.486 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi123.front.sepia.ceph.com 2024-09-21T10:36:09.486 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -- bash -c 'ceph orch status' 2024-09-21T10:36:09.981 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:09 smithi123 ceph-mon[23709]: from='client.? 172.21.15.123:0/1179144343' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-21T10:36:10.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:09 smithi175 ceph-mon[25806]: from='client.? 172.21.15.123:0/1179144343' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-21T10:36:11.029 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:10 smithi123 ceph-mon[23709]: pgmap v209: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:36:11.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:10 smithi175 ceph-mon[25806]: pgmap v209: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:36:11.245 INFO:teuthology.orchestra.run.smithi123.stdout:Backend: cephadm 2024-09-21T10:36:11.246 INFO:teuthology.orchestra.run.smithi123.stdout:Available: Yes 2024-09-21T10:36:11.246 INFO:teuthology.orchestra.run.smithi123.stdout:Paused: No 2024-09-21T10:36:11.824 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:11 smithi123 ceph-mon[23709]: from='client.14464 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:36:11.847 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -- bash -c 'ceph orch ps' 2024-09-21T10:36:12.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:11 smithi175 ceph-mon[25806]: from='client.14464 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:36:13.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:12 smithi123 ceph-mon[23709]: pgmap v210: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:36:13.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:12 smithi175 ceph-mon[25806]: pgmap v210: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:36:13.724 INFO:teuthology.orchestra.run.smithi123.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T10:36:13.724 INFO:teuthology.orchestra.run.smithi123.stdout:alertmanager.smithi123 smithi123 *:9093,9094 running (4m) 2m ago 5m 15.5M - ba2b418f427c db0d7d5d885a 2024-09-21T10:36:13.724 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi123 smithi123 running (5m) 2m ago 5m 6405k - 17.2.0 e1d6a67b021e c459f87d5b50 2024-09-21T10:36:13.724 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi175 smithi175 running (5m) 71s ago 5m 6405k - 17.2.0 e1d6a67b021e e35e422d5583 2024-09-21T10:36:13.724 INFO:teuthology.orchestra.run.smithi123.stdout:grafana.smithi123 smithi123 *:3000 running (4m) 2m ago 5m 45.2M - 8.3.5 dad864ee21e9 86ee7306dede 2024-09-21T10:36:13.724 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi123.stnuit smithi123 *:9283 running (7m) 2m ago 7m 447M - 17.2.0 e1d6a67b021e 417fa41b561b 2024-09-21T10:36:13.724 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi175.txvwrq smithi175 *:8443,9283 running (5m) 71s ago 5m 410M - 17.2.0 e1d6a67b021e 79b9f39b87a4 2024-09-21T10:36:13.724 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi123 smithi123 running (7m) 2m ago 7m 41.5M 2048M 17.2.0 e1d6a67b021e 6ad9a8bc53b7 2024-09-21T10:36:13.724 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi175 smithi175 running (5m) 71s ago 5m 42.0M 2048M 17.2.0 e1d6a67b021e 753f564c8cf5 2024-09-21T10:36:13.724 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi123 smithi123 *:9100 running (5m) 2m ago 5m 18.4M - 1dbe0e931976 875d5b2d6936 2024-09-21T10:36:13.724 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi175 smithi175 *:9100 running (4m) 71s ago 4m 17.4M - 1dbe0e931976 92ddc27e1ca2 2024-09-21T10:36:13.725 INFO:teuthology.orchestra.run.smithi123.stdout:osd.0 smithi123 running (4m) 2m ago 4m 33.1M 3216M 17.2.0 e1d6a67b021e a94edf88c456 2024-09-21T10:36:13.725 INFO:teuthology.orchestra.run.smithi123.stdout:osd.1 smithi123 running (3m) 2m ago 3m 37.3M 3216M 17.2.0 e1d6a67b021e 59aee515b0f8 2024-09-21T10:36:13.725 INFO:teuthology.orchestra.run.smithi123.stdout:osd.2 smithi123 running (3m) 2m ago 3m 31.9M 3216M 17.2.0 e1d6a67b021e 7db93f8ce5b5 2024-09-21T10:36:13.725 INFO:teuthology.orchestra.run.smithi123.stdout:osd.3 smithi123 running (2m) 2m ago 2m 10.4M 3216M 17.2.0 e1d6a67b021e c1672dea7b1f 2024-09-21T10:36:13.725 INFO:teuthology.orchestra.run.smithi123.stdout:osd.4 smithi175 running (2m) 71s ago 2m 32.9M 3984M 17.2.0 e1d6a67b021e 48fe728c259e 2024-09-21T10:36:13.725 INFO:teuthology.orchestra.run.smithi123.stdout:osd.5 smithi175 running (2m) 71s ago 2m 34.3M 3984M 17.2.0 e1d6a67b021e e270d5a21b14 2024-09-21T10:36:13.725 INFO:teuthology.orchestra.run.smithi123.stdout:osd.6 smithi175 running (100s) 71s ago 100s 29.9M 3984M 17.2.0 e1d6a67b021e 9bcbf018eb20 2024-09-21T10:36:13.725 INFO:teuthology.orchestra.run.smithi123.stdout:osd.7 smithi175 running (76s) 71s ago 75s 10.0M 3984M 17.2.0 e1d6a67b021e e9b8f1f90994 2024-09-21T10:36:13.725 INFO:teuthology.orchestra.run.smithi123.stdout:prometheus.smithi123 smithi123 *:9095 running (4m) 2m ago 4m 49.1M - 514e6a882f6e 2beddd4ece2e 2024-09-21T10:36:14.404 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -- bash -c 'ceph orch ls' 2024-09-21T10:36:14.950 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:14 smithi123 ceph-mon[23709]: pgmap v211: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:36:14.950 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:14 smithi123 ceph-mon[23709]: from='client.14468 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:36:15.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:14 smithi175 ceph-mon[25806]: pgmap v211: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:36:15.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:14 smithi175 ceph-mon[25806]: from='client.14468 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:36:16.423 INFO:teuthology.orchestra.run.smithi123.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-09-21T10:36:16.423 INFO:teuthology.orchestra.run.smithi123.stdout:alertmanager ?:9093,9094 1/1 2m ago 6m count:1 2024-09-21T10:36:16.423 INFO:teuthology.orchestra.run.smithi123.stdout:crash 2/2 2m ago 6m * 2024-09-21T10:36:16.423 INFO:teuthology.orchestra.run.smithi123.stdout:grafana ?:3000 1/1 2m ago 6m count:1 2024-09-21T10:36:16.423 INFO:teuthology.orchestra.run.smithi123.stdout:mgr 2/2 2m ago 6m count:2 2024-09-21T10:36:16.423 INFO:teuthology.orchestra.run.smithi123.stdout:mon 2/2 2m ago 5m smithi123:172.21.15.123=smithi123;smithi175:172.21.15.175=smithi175;count:2 2024-09-21T10:36:16.423 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter ?:9100 2/2 2m ago 6m * 2024-09-21T10:36:16.423 INFO:teuthology.orchestra.run.smithi123.stdout:osd 8 2m ago - 2024-09-21T10:36:16.423 INFO:teuthology.orchestra.run.smithi123.stdout:prometheus ?:9095 1/1 2m ago 6m count:1 2024-09-21T10:36:17.027 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:16 smithi123 ceph-mon[23709]: pgmap v212: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:36:17.050 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -- bash -c 'ceph orch host ls' 2024-09-21T10:36:17.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:16 smithi175 ceph-mon[25806]: pgmap v212: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:36:18.000 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:17 smithi123 ceph-mon[23709]: from='client.14472 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:36:18.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:17 smithi175 ceph-mon[25806]: from='client.14472 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:36:19.048 INFO:teuthology.orchestra.run.smithi123.stdout:HOST ADDR LABELS STATUS 2024-09-21T10:36:19.048 INFO:teuthology.orchestra.run.smithi123.stdout:smithi123 172.21.15.123 2024-09-21T10:36:19.049 INFO:teuthology.orchestra.run.smithi123.stdout:smithi175 172.21.15.175 2024-09-21T10:36:19.049 INFO:teuthology.orchestra.run.smithi123.stdout:2 hosts in cluster 2024-09-21T10:36:19.049 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:18 smithi123 ceph-mon[23709]: pgmap v213: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:36:19.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:18 smithi175 ceph-mon[25806]: pgmap v213: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:36:19.641 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -- bash -c 'ceph orch device ls' 2024-09-21T10:36:19.850 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:19 smithi123 ceph-mon[23709]: from='client.14476 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:36:20.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:19 smithi175 ceph-mon[25806]: from='client.14476 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:36:21.048 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:20 smithi123 ceph-mon[23709]: pgmap v214: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:36:21.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:20 smithi175 ceph-mon[25806]: pgmap v214: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:36:21.451 INFO:teuthology.orchestra.run.smithi123.stdout:HOST PATH TYPE DEVICE ID SIZE AVAILABLE REJECT REASONS 2024-09-21T10:36:21.451 INFO:teuthology.orchestra.run.smithi123.stdout:smithi123 /dev/nvme0n1 ssd INTEL_SSDPEDMD400G4_PHFT620400YK400BGN 400G LVM detected, locked 2024-09-21T10:36:21.452 INFO:teuthology.orchestra.run.smithi123.stdout:smithi175 /dev/nvme0n1 ssd INTEL_SSDPEDMD400G4_CVFT623300M0400BGN 400G LVM detected, locked 2024-09-21T10:36:22.089 INFO:teuthology.run_tasks:Running task vip.exec... 2024-09-21T10:36:22.102 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi123.front.sepia.ceph.com 2024-09-21T10:36:22.102 DEBUG:teuthology.orchestra.run.smithi123:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'systemctl stop nfs-server' 2024-09-21T10:36:22.130 INFO:teuthology.orchestra.run.smithi123.stderr:+ systemctl stop nfs-server 2024-09-21T10:36:22.139 INFO:tasks.vip:Running commands on role host.b host ubuntu@smithi175.front.sepia.ceph.com 2024-09-21T10:36:22.139 DEBUG:teuthology.orchestra.run.smithi175:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'systemctl stop nfs-server' 2024-09-21T10:36:22.169 INFO:teuthology.orchestra.run.smithi175.stderr:+ systemctl stop nfs-server 2024-09-21T10:36:22.177 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-21T10:36:22.257 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi123.front.sepia.ceph.com 2024-09-21T10:36:22.258 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -- bash -c 'ceph fs volume create foofs' 2024-09-21T10:36:22.825 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:22 smithi123 ceph-mon[23709]: from='client.14480 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:36:22.826 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:22 smithi123 ceph-mon[23709]: pgmap v215: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:36:23.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:22 smithi175 ceph-mon[25806]: from='client.14480 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:36:23.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:22 smithi175 ceph-mon[25806]: pgmap v215: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:36:25.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:24 smithi123 ceph-mon[23709]: pgmap v216: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:36:25.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:24 smithi123 ceph-mon[23709]: from='client.14484 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foofs", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:36:25.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:24 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]: dispatch 2024-09-21T10:36:25.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:24 smithi175 ceph-mon[25806]: pgmap v216: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:36:25.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:24 smithi175 ceph-mon[25806]: from='client.14484 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foofs", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:36:25.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:24 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]: dispatch 2024-09-21T10:36:26.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:25 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]': finished 2024-09-21T10:36:26.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:25 smithi123 ceph-mon[23709]: osdmap e61: 8 total, 8 up, 8 in 2024-09-21T10:36:26.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:25 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]: dispatch 2024-09-21T10:36:26.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:25 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-mon-smithi123[23705]: 2024-09-21T10:36:25.765+0000 7fa31e266700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-21T10:36:26.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:25 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]': finished 2024-09-21T10:36:26.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:25 smithi175 ceph-mon[25806]: osdmap e61: 8 total, 8 up, 8 in 2024-09-21T10:36:26.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:25 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]: dispatch 2024-09-21T10:36:26.400 INFO:teuthology.run_tasks:Running task cephadm.wait_for_service... 2024-09-21T10:36:26.409 INFO:tasks.cephadm:Waiting for ceph service mds.foofs to start (timeout 300)... 2024-09-21T10:36:26.409 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -- ceph orch ls -f json 2024-09-21T10:36:26.959 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:26 smithi123 ceph-mon[23709]: pgmap v218: 33 pgs: 17 creating+peering, 15 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:36:26.959 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:26 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]': finished 2024-09-21T10:36:26.959 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:26 smithi123 ceph-mon[23709]: osdmap e62: 8 total, 8 up, 8 in 2024-09-21T10:36:26.960 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:26 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]: dispatch 2024-09-21T10:36:26.960 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:26 smithi123 ceph-mon[23709]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-21T10:36:26.960 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:26 smithi123 ceph-mon[23709]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-09-21T10:36:26.960 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:26 smithi123 ceph-mon[23709]: osdmap e63: 8 total, 8 up, 8 in 2024-09-21T10:36:26.960 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:26 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]': finished 2024-09-21T10:36:26.960 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:26 smithi123 ceph-mon[23709]: fsmap foofs:0 2024-09-21T10:36:26.960 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:26 smithi123 ceph-mon[23709]: Saving service mds.foofs spec with placement count:2 2024-09-21T10:36:26.960 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:26 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:36:26.960 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:26 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:36:26.960 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:26 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:36:26.960 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:26 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T10:36:26.960 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:26 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:36:26.961 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:26 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:36:26.961 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:26 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi123.uqoyow", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-21T10:36:26.961 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:26 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi123.uqoyow", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-21T10:36:26.961 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:26 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:36:26.961 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:26 smithi123 ceph-mon[23709]: Deploying daemon mds.foofs.smithi123.uqoyow on smithi123 2024-09-21T10:36:27.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:26 smithi175 ceph-mon[25806]: pgmap v218: 33 pgs: 17 creating+peering, 15 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:36:27.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:26 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]': finished 2024-09-21T10:36:27.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:26 smithi175 ceph-mon[25806]: osdmap e62: 8 total, 8 up, 8 in 2024-09-21T10:36:27.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:26 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]: dispatch 2024-09-21T10:36:27.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:26 smithi175 ceph-mon[25806]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-21T10:36:27.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:26 smithi175 ceph-mon[25806]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-09-21T10:36:27.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:26 smithi175 ceph-mon[25806]: osdmap e63: 8 total, 8 up, 8 in 2024-09-21T10:36:27.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:26 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]': finished 2024-09-21T10:36:27.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:26 smithi175 ceph-mon[25806]: fsmap foofs:0 2024-09-21T10:36:27.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:26 smithi175 ceph-mon[25806]: Saving service mds.foofs spec with placement count:2 2024-09-21T10:36:27.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:26 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:36:27.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:26 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:36:27.125 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:26 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:36:27.125 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:26 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T10:36:27.125 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:26 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:36:27.125 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:26 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:36:27.125 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:26 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi123.uqoyow", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-21T10:36:27.125 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:26 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi123.uqoyow", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-21T10:36:27.125 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:26 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:36:27.125 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:26 smithi175 ceph-mon[25806]: Deploying daemon mds.foofs.smithi123.uqoyow on smithi123 2024-09-21T10:36:28.015 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:27 smithi123 ceph-mon[23709]: osdmap e64: 8 total, 8 up, 8 in 2024-09-21T10:36:28.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:27 smithi175 ceph-mon[25806]: osdmap e64: 8 total, 8 up, 8 in 2024-09-21T10:36:29.030 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:28 smithi123 ceph-mon[23709]: pgmap v222: 65 pgs: 21 creating+peering, 34 unknown, 10 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:36:29.030 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:28 smithi123 ceph-mon[23709]: osdmap e65: 8 total, 8 up, 8 in 2024-09-21T10:36:29.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:28 smithi175 ceph-mon[25806]: pgmap v222: 65 pgs: 21 creating+peering, 34 unknown, 10 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:36:29.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:28 smithi175 ceph-mon[25806]: osdmap e65: 8 total, 8 up, 8 in 2024-09-21T10:36:29.575 INFO:teuthology.orchestra.run.smithi123.stdout: 2024-09-21T10:36:29.575 INFO:teuthology.orchestra.run.smithi123.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-21T10:29:47.207278Z", "last_refresh": "2024-09-21T10:33:28.681345Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-21T10:29:40.698729Z", "last_refresh": "2024-09-21T10:33:28.681407Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-09-21T10:29:44.033793Z", "last_refresh": "2024-09-21T10:33:28.681467Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-09-21T10:36:25.790132Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-21T10:36:25.784744Z", "running": 0, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-21T10:29:38.874687Z", "last_refresh": "2024-09-21T10:33:28.681276Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi123:172.21.15.123=smithi123", "smithi175:172.21.15.175=smithi175"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-21T10:30:30.797624Z", "last_refresh": "2024-09-21T10:33:28.681121Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-21T10:29:45.863421Z", "last_refresh": "2024-09-21T10:33:28.681525Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-09-21T10:33:28.681643Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-21T10:29:42.699451Z", "last_refresh": "2024-09-21T10:33:28.681584Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-21T10:36:30.435 INFO:tasks.cephadm:mds.foofs has 0/2 2024-09-21T10:36:31.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:31 smithi123 ceph-mon[23709]: from='client.14488 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:36:31.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:31 smithi123 ceph-mon[23709]: pgmap v224: 65 pgs: 21 creating+peering, 21 unknown, 23 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:36:31.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:31 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:36:31.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:31 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi175.jrlevw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-21T10:36:31.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:31 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi175.jrlevw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-21T10:36:31.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:31 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:36:31.352 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:31 smithi123 ceph-mon[23709]: Deploying daemon mds.foofs.smithi175.jrlevw on smithi175 2024-09-21T10:36:31.352 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:31 smithi123 ceph-mon[23709]: daemon mds.foofs.smithi123.uqoyow assigned to filesystem foofs as rank 0 (now has 1 ranks) 2024-09-21T10:36:31.352 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:31 smithi123 ceph-mon[23709]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-21T10:36:31.352 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:31 smithi123 ceph-mon[23709]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-09-21T10:36:31.352 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:31 smithi123 ceph-mon[23709]: Cluster is now healthy 2024-09-21T10:36:31.352 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:31 smithi123 ceph-mon[23709]: mds.? [v2:172.21.15.123:6834/625956046,v1:172.21.15.123:6835/625956046] up:boot 2024-09-21T10:36:31.352 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:31 smithi123 ceph-mon[23709]: fsmap foofs:1 {0=foofs.smithi123.uqoyow=up:creating} 2024-09-21T10:36:31.352 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:31 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi123.uqoyow"}]: dispatch 2024-09-21T10:36:31.352 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:31 smithi123 ceph-mon[23709]: daemon mds.foofs.smithi123.uqoyow is now active in filesystem foofs as rank 0 2024-09-21T10:36:31.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:31 smithi175 ceph-mon[25806]: from='client.14488 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:36:31.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:31 smithi175 ceph-mon[25806]: pgmap v224: 65 pgs: 21 creating+peering, 21 unknown, 23 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:36:31.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:31 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:36:31.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:31 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi175.jrlevw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-21T10:36:31.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:31 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi175.jrlevw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-21T10:36:31.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:31 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:36:31.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:31 smithi175 ceph-mon[25806]: Deploying daemon mds.foofs.smithi175.jrlevw on smithi175 2024-09-21T10:36:31.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:31 smithi175 ceph-mon[25806]: daemon mds.foofs.smithi123.uqoyow assigned to filesystem foofs as rank 0 (now has 1 ranks) 2024-09-21T10:36:31.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:31 smithi175 ceph-mon[25806]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-21T10:36:31.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:31 smithi175 ceph-mon[25806]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-09-21T10:36:31.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:31 smithi175 ceph-mon[25806]: Cluster is now healthy 2024-09-21T10:36:31.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:31 smithi175 ceph-mon[25806]: mds.? [v2:172.21.15.123:6834/625956046,v1:172.21.15.123:6835/625956046] up:boot 2024-09-21T10:36:31.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:31 smithi175 ceph-mon[25806]: fsmap foofs:1 {0=foofs.smithi123.uqoyow=up:creating} 2024-09-21T10:36:31.375 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:31 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi123.uqoyow"}]: dispatch 2024-09-21T10:36:31.375 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:31 smithi175 ceph-mon[25806]: daemon mds.foofs.smithi123.uqoyow is now active in filesystem foofs as rank 0 2024-09-21T10:36:31.435 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -- ceph orch ls -f json 2024-09-21T10:36:32.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:32 smithi123 ceph-mon[23709]: pgmap v225: 65 pgs: 65 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 174 B/s wr, 0 op/s 2024-09-21T10:36:32.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:32 smithi123 ceph-mon[23709]: mds.? [v2:172.21.15.123:6834/625956046,v1:172.21.15.123:6835/625956046] up:active 2024-09-21T10:36:32.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:32 smithi123 ceph-mon[23709]: fsmap foofs:1 {0=foofs.smithi123.uqoyow=up:active} 2024-09-21T10:36:32.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:32 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:36:32.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:32 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:36:32.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:32 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:36:32.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:32 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T10:36:32.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:32 smithi175 ceph-mon[25806]: pgmap v225: 65 pgs: 65 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 174 B/s wr, 0 op/s 2024-09-21T10:36:32.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:32 smithi175 ceph-mon[25806]: mds.? [v2:172.21.15.123:6834/625956046,v1:172.21.15.123:6835/625956046] up:active 2024-09-21T10:36:32.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:32 smithi175 ceph-mon[25806]: fsmap foofs:1 {0=foofs.smithi123.uqoyow=up:active} 2024-09-21T10:36:32.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:32 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:36:32.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:32 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:36:32.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:32 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:36:32.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:32 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T10:36:33.546 INFO:teuthology.orchestra.run.smithi123.stdout: 2024-09-21T10:36:33.546 INFO:teuthology.orchestra.run.smithi123.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-21T10:29:47.207278Z", "last_refresh": "2024-09-21T10:33:28.681345Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-21T10:29:40.698729Z", "last_refresh": "2024-09-21T10:33:28.681407Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-09-21T10:29:44.033793Z", "last_refresh": "2024-09-21T10:33:28.681467Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-09-21T10:36:25.790132Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-21T10:36:25.784744Z", "running": 0, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-21T10:29:38.874687Z", "last_refresh": "2024-09-21T10:33:28.681276Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi123:172.21.15.123=smithi123", "smithi175:172.21.15.175=smithi175"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-21T10:30:30.797624Z", "last_refresh": "2024-09-21T10:33:28.681121Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-21T10:29:45.863421Z", "last_refresh": "2024-09-21T10:33:28.681525Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-09-21T10:33:28.681643Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-21T10:29:42.699451Z", "last_refresh": "2024-09-21T10:33:28.681584Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-21T10:36:33.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:33 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:36:33.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:33 smithi123 ceph-mon[23709]: mds.? [v2:172.21.15.175:6832/3454320220,v1:172.21.15.175:6833/3454320220] up:boot 2024-09-21T10:36:33.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:33 smithi123 ceph-mon[23709]: fsmap foofs:1 {0=foofs.smithi123.uqoyow=up:active} 1 up:standby 2024-09-21T10:36:33.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:33 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi175.jrlevw"}]: dispatch 2024-09-21T10:36:33.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:33 smithi123 ceph-mon[23709]: fsmap foofs:1 {0=foofs.smithi123.uqoyow=up:active} 1 up:standby 2024-09-21T10:36:33.898 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:33 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:36:33.898 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:33 smithi175 ceph-mon[25806]: mds.? [v2:172.21.15.175:6832/3454320220,v1:172.21.15.175:6833/3454320220] up:boot 2024-09-21T10:36:33.898 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:33 smithi175 ceph-mon[25806]: fsmap foofs:1 {0=foofs.smithi123.uqoyow=up:active} 1 up:standby 2024-09-21T10:36:33.898 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:33 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi175.jrlevw"}]: dispatch 2024-09-21T10:36:33.898 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:33 smithi175 ceph-mon[25806]: fsmap foofs:1 {0=foofs.smithi123.uqoyow=up:active} 1 up:standby 2024-09-21T10:36:34.575 INFO:tasks.cephadm:mds.foofs has 0/2 2024-09-21T10:36:35.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:35 smithi123 ceph-mon[23709]: from='client.14498 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:36:35.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:35 smithi123 ceph-mon[23709]: pgmap v226: 65 pgs: 65 active+clean; 577 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s wr, 2 op/s 2024-09-21T10:36:35.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:35 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:36:35.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:35 smithi175 ceph-mon[25806]: from='client.14498 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:36:35.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:35 smithi175 ceph-mon[25806]: pgmap v226: 65 pgs: 65 active+clean; 577 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s wr, 2 op/s 2024-09-21T10:36:35.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:35 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:36:35.576 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -- ceph orch ls -f json 2024-09-21T10:36:37.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:37 smithi123 ceph-mon[23709]: pgmap v227: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s wr, 5 op/s 2024-09-21T10:36:37.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:37 smithi175 ceph-mon[25806]: pgmap v227: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s wr, 5 op/s 2024-09-21T10:36:38.942 INFO:teuthology.orchestra.run.smithi123.stdout: 2024-09-21T10:36:38.942 INFO:teuthology.orchestra.run.smithi123.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-21T10:29:47.207278Z", "last_refresh": "2024-09-21T10:36:38.831263Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-21T10:29:40.698729Z", "last_refresh": "2024-09-21T10:36:34.563488Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-09-21T10:29:44.033793Z", "last_refresh": "2024-09-21T10:36:38.831392Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-09-21T10:36:25.790132Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-21T10:36:25.784744Z", "last_refresh": "2024-09-21T10:36:34.564576Z", "running": 2, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-21T10:29:38.874687Z", "last_refresh": "2024-09-21T10:36:34.563705Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi123:172.21.15.123=smithi123", "smithi175:172.21.15.175=smithi175"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-21T10:30:30.797624Z", "last_refresh": "2024-09-21T10:36:34.563839Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-21T10:29:45.863421Z", "last_refresh": "2024-09-21T10:36:34.563971Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-09-21T10:36:34.564095Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-21T10:29:42.699451Z", "last_refresh": "2024-09-21T10:36:38.831517Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-21T10:36:38.942 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:38 smithi123 ceph-mon[23709]: pgmap v228: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-09-21T10:36:38.942 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:38 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:36:39.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:38 smithi175 ceph-mon[25806]: pgmap v228: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-09-21T10:36:39.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:38 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:36:39.651 INFO:tasks.cephadm:mds.foofs has 2/2 2024-09-21T10:36:39.651 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-21T10:36:39.661 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi123.front.sepia.ceph.com 2024-09-21T10:36:39.661 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -- bash -c 'ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2' 2024-09-21T10:36:39.970 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:39 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:36:39.970 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:39 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:36:39.970 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:39 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:36:39.970 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:39 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:36:39.970 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:39 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T10:36:39.971 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:39 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:36:39.971 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:39 smithi123 ceph-mon[23709]: from='client.14502 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:36:40.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:39 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:36:40.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:39 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:36:40.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:39 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:36:40.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:39 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:36:40.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:39 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T10:36:40.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:39 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:36:40.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:39 smithi175 ceph-mon[25806]: from='client.14502 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:36:41.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:40 smithi175 ceph-mon[25806]: pgmap v229: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-21T10:36:41.165 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:40 smithi123 ceph-mon[23709]: pgmap v229: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-21T10:36:42.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:41 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd pool create", "pool": ".nfs"}]: dispatch 2024-09-21T10:36:42.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:41 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd pool create", "pool": ".nfs"}]: dispatch 2024-09-21T10:36:42.892 INFO:teuthology.orchestra.run.smithi123.stdout:NFS Cluster Created Successfully 2024-09-21T10:36:43.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:42 smithi175 ceph-mon[25806]: from='client.14506 -' entity='client.admin' cmd=[{"prefix": "nfs cluster create", "cluster_id": "foo", "placement": "2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:36:43.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:42 smithi175 ceph-mon[25806]: pgmap v230: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-21T10:36:43.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:42 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "osd pool create", "pool": ".nfs"}]': finished 2024-09-21T10:36:43.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:42 smithi175 ceph-mon[25806]: osdmap e66: 8 total, 8 up, 8 in 2024-09-21T10:36:43.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:42 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2024-09-21T10:36:43.208 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:42 smithi123 ceph-mon[23709]: from='client.14506 -' entity='client.admin' cmd=[{"prefix": "nfs cluster create", "cluster_id": "foo", "placement": "2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:36:43.208 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:42 smithi123 ceph-mon[23709]: pgmap v230: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-21T10:36:43.208 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:42 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "osd pool create", "pool": ".nfs"}]': finished 2024-09-21T10:36:43.208 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:42 smithi123 ceph-mon[23709]: osdmap e66: 8 total, 8 up, 8 in 2024-09-21T10:36:43.209 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:42 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2024-09-21T10:36:43.623 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -- bash -c 'ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake' 2024-09-21T10:36:44.171 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:43 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]': finished 2024-09-21T10:36:44.172 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:43 smithi123 ceph-mon[23709]: osdmap e67: 8 total, 8 up, 8 in 2024-09-21T10:36:44.172 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:43 smithi123 ceph-mon[23709]: Saving service nfs.foo spec with placement count:2 2024-09-21T10:36:44.172 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:43 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:36:44.172 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:43 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:36:44.172 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:43 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:36:44.172 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:43 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T10:36:44.172 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:43 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:36:44.172 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:43 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:36:44.172 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:43 smithi123 ceph-mon[23709]: Creating key for client.nfs.foo.0.0.smithi123.ddatpr 2024-09-21T10:36:44.172 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:43 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi123.ddatpr", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-21T10:36:44.172 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:43 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi123.ddatpr", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-09-21T10:36:44.172 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:43 smithi123 ceph-mon[23709]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-09-21T10:36:44.173 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:43 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-21T10:36:44.173 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:43 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-21T10:36:44.173 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:43 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:36:44.173 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:43 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-21T10:36:44.173 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:43 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-21T10:36:44.173 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:43 smithi123 ceph-mon[23709]: Rados config object exists: conf-nfs.foo 2024-09-21T10:36:44.173 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:43 smithi123 ceph-mon[23709]: Creating key for client.nfs.foo.0.0.smithi123.ddatpr-rgw 2024-09-21T10:36:44.173 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:43 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi123.ddatpr-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-21T10:36:44.173 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:43 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi123.ddatpr-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-21T10:36:44.173 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:43 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:36:44.173 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:43 smithi123 ceph-mon[23709]: Deploying daemon nfs.foo.0.0.smithi123.ddatpr on smithi123 2024-09-21T10:36:44.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:43 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]': finished 2024-09-21T10:36:44.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:43 smithi175 ceph-mon[25806]: osdmap e67: 8 total, 8 up, 8 in 2024-09-21T10:36:44.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:43 smithi175 ceph-mon[25806]: Saving service nfs.foo spec with placement count:2 2024-09-21T10:36:44.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:43 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:36:44.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:43 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:36:44.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:43 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:36:44.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:43 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T10:36:44.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:43 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:36:44.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:43 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:36:44.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:43 smithi175 ceph-mon[25806]: Creating key for client.nfs.foo.0.0.smithi123.ddatpr 2024-09-21T10:36:44.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:43 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi123.ddatpr", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-21T10:36:44.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:43 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi123.ddatpr", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-09-21T10:36:44.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:43 smithi175 ceph-mon[25806]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-09-21T10:36:44.375 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:43 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-21T10:36:44.375 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:43 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-21T10:36:44.375 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:43 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:36:44.375 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:43 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-21T10:36:44.375 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:43 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-21T10:36:44.375 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:43 smithi175 ceph-mon[25806]: Rados config object exists: conf-nfs.foo 2024-09-21T10:36:44.375 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:43 smithi175 ceph-mon[25806]: Creating key for client.nfs.foo.0.0.smithi123.ddatpr-rgw 2024-09-21T10:36:44.375 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:43 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi123.ddatpr-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-21T10:36:44.375 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:43 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi123.ddatpr-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-21T10:36:44.375 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:43 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:36:44.375 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:43 smithi175 ceph-mon[25806]: Deploying daemon nfs.foo.0.0.smithi123.ddatpr on smithi123 2024-09-21T10:36:45.156 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:44 smithi123 ceph-mon[23709]: pgmap v233: 97 pgs: 5 creating+peering, 26 unknown, 66 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:36:45.156 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:44 smithi123 ceph-mon[23709]: osdmap e68: 8 total, 8 up, 8 in 2024-09-21T10:36:45.156 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:44 smithi123 ceph-mon[23709]: mgrmap e20: smithi123.stnuit(active, since 5m), standbys: smithi175.txvwrq 2024-09-21T10:36:45.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:44 smithi175 ceph-mon[25806]: pgmap v233: 97 pgs: 5 creating+peering, 26 unknown, 66 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-09-21T10:36:45.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:44 smithi175 ceph-mon[25806]: osdmap e68: 8 total, 8 up, 8 in 2024-09-21T10:36:45.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:44 smithi175 ceph-mon[25806]: mgrmap e20: smithi123.stnuit(active, since 5m), standbys: smithi175.txvwrq 2024-09-21T10:36:47.020 INFO:teuthology.orchestra.run.smithi123.stderr:Invalid command: Unexpected argument '--clusterid' 2024-09-21T10:36:47.020 INFO:teuthology.orchestra.run.smithi123.stderr:nfs export create cephfs [] [--readonly] [--client_addr ...] [--squash ] : Create a CephFS export 2024-09-21T10:36:47.020 INFO:teuthology.orchestra.run.smithi123.stderr:Error EINVAL: invalid command 2024-09-21T10:36:47.166 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:46 smithi123 ceph-mon[23709]: pgmap v235: 97 pgs: 5 creating+peering, 9 unknown, 83 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-09-21T10:36:47.371 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-09-21T10:36:47.372 INFO:teuthology.orchestra.run.smithi123.stdout: "bind": "/fake", 2024-09-21T10:36:47.372 INFO:teuthology.orchestra.run.smithi123.stdout: "fs": "foofs", 2024-09-21T10:36:47.372 INFO:teuthology.orchestra.run.smithi123.stdout: "path": "/", 2024-09-21T10:36:47.372 INFO:teuthology.orchestra.run.smithi123.stdout: "cluster": "foo", 2024-09-21T10:36:47.372 INFO:teuthology.orchestra.run.smithi123.stdout: "mode": "RW" 2024-09-21T10:36:47.372 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-09-21T10:36:47.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:46 smithi175 ceph-mon[25806]: pgmap v235: 97 pgs: 5 creating+peering, 9 unknown, 83 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-09-21T10:36:48.111 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -- bash -c 'while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done' 2024-09-21T10:36:48.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:48 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:36:48.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:48 smithi123 ceph-mon[23709]: Creating key for client.nfs.foo.1.0.smithi175.wpawer 2024-09-21T10:36:48.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:48 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi175.wpawer", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-21T10:36:48.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:48 smithi123 ceph-mon[23709]: from='client.14528 -' entity='client.admin' cmd=[{"prefix": "nfs export create cephfs", "fsname": "foofs", "cluster_id": "foo", "pseudo_path": "/fake", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:36:48.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:48 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi175.wpawer", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-09-21T10:36:48.352 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:48 smithi123 ceph-mon[23709]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-09-21T10:36:48.352 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:48 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-21T10:36:48.352 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:48 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-21T10:36:48.352 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:48 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:36:48.352 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:48 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-09-21T10:36:48.352 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:48 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-09-21T10:36:48.352 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:48 smithi123 ceph-mon[23709]: pgmap v236: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-09-21T10:36:48.352 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:48 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.stnuit/mirror_snapshot_schedule"}]: dispatch 2024-09-21T10:36:48.352 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:48 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.stnuit/trash_purge_schedule"}]: dispatch 2024-09-21T10:36:48.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:48 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:36:48.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:48 smithi175 ceph-mon[25806]: Creating key for client.nfs.foo.1.0.smithi175.wpawer 2024-09-21T10:36:48.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:48 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi175.wpawer", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-21T10:36:48.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:48 smithi175 ceph-mon[25806]: from='client.14528 -' entity='client.admin' cmd=[{"prefix": "nfs export create cephfs", "fsname": "foofs", "cluster_id": "foo", "pseudo_path": "/fake", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:36:48.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:48 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi175.wpawer", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-09-21T10:36:48.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:48 smithi175 ceph-mon[25806]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-09-21T10:36:48.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:48 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-21T10:36:48.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:48 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-21T10:36:48.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:48 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:36:48.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:48 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-09-21T10:36:48.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:48 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-09-21T10:36:48.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:48 smithi175 ceph-mon[25806]: pgmap v236: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-09-21T10:36:48.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:48 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.stnuit/mirror_snapshot_schedule"}]: dispatch 2024-09-21T10:36:48.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:48 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.stnuit/trash_purge_schedule"}]: dispatch 2024-09-21T10:36:50.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:50 smithi123 ceph-mon[23709]: pgmap v237: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1.3 KiB/s wr, 3 op/s 2024-09-21T10:36:50.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:50 smithi123 ceph-mon[23709]: from='client.14544 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:36:50.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:50 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-21T10:36:50.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:50 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-21T10:36:50.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:50 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi175.wpawer-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-21T10:36:50.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:50 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi175.wpawer-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-21T10:36:50.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:50 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:36:50.948 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:50 smithi175 ceph-mon[25806]: pgmap v237: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1.3 KiB/s wr, 3 op/s 2024-09-21T10:36:50.948 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:50 smithi175 ceph-mon[25806]: from='client.14544 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:36:50.948 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:50 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-21T10:36:50.948 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:50 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-21T10:36:50.948 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:50 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi175.wpawer-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-21T10:36:50.948 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:50 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi175.wpawer-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-21T10:36:50.948 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:50 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:36:52.012 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:51 smithi175 ceph-mon[25806]: Rados config object exists: conf-nfs.foo 2024-09-21T10:36:52.013 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:51 smithi175 ceph-mon[25806]: Creating key for client.nfs.foo.1.0.smithi175.wpawer-rgw 2024-09-21T10:36:52.013 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:51 smithi175 ceph-mon[25806]: Deploying daemon nfs.foo.1.0.smithi175.wpawer on smithi175 2024-09-21T10:36:52.100 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:51 smithi123 ceph-mon[23709]: Rados config object exists: conf-nfs.foo 2024-09-21T10:36:52.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:51 smithi123 ceph-mon[23709]: Creating key for client.nfs.foo.1.0.smithi175.wpawer-rgw 2024-09-21T10:36:52.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:51 smithi123 ceph-mon[23709]: Deploying daemon nfs.foo.1.0.smithi175.wpawer on smithi175 2024-09-21T10:36:53.100 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:52 smithi123 ceph-mon[23709]: from='client.14558 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:36:53.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:52 smithi123 ceph-mon[23709]: pgmap v238: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1.3 KiB/s wr, 2 op/s 2024-09-21T10:36:53.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:52 smithi175 ceph-mon[25806]: from='client.14558 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:36:53.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:52 smithi175 ceph-mon[25806]: pgmap v238: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1.3 KiB/s wr, 2 op/s 2024-09-21T10:36:54.219 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:53 smithi123 ceph-mon[23709]: from='client.14562 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:36:54.219 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:53 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:36:54.219 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:53 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:36:54.220 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:53 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:36:54.220 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:53 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T10:36:54.263 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:53 smithi175 ceph-mon[25806]: from='client.14562 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:36:54.263 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:53 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:36:54.263 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:53 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:36:54.263 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:53 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:36:54.263 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:53 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T10:36:55.158 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:54 smithi175 ceph-mon[25806]: pgmap v239: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1.1 KiB/s wr, 2 op/s 2024-09-21T10:36:55.158 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:54 smithi175 ceph-mon[25806]: from='client.14578 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:36:55.350 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:54 smithi123 ceph-mon[23709]: pgmap v239: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1.1 KiB/s wr, 2 op/s 2024-09-21T10:36:55.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:54 smithi123 ceph-mon[23709]: from='client.14578 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:36:56.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:56 smithi123 ceph-mon[23709]: from='client.14582 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:36:56.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:56 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:36:56.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:56 smithi123 ceph-mon[23709]: pgmap v240: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.6 KiB/s rd, 2.3 KiB/s wr, 6 op/s 2024-09-21T10:36:56.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:56 smithi175 ceph-mon[25806]: from='client.14582 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:36:56.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:56 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:36:56.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:56 smithi175 ceph-mon[25806]: pgmap v240: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.6 KiB/s rd, 2.3 KiB/s wr, 6 op/s 2024-09-21T10:36:57.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:57 smithi123 ceph-mon[23709]: from='client.14586 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:36:57.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:57 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:36:57.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:57 smithi175 ceph-mon[25806]: from='client.14586 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:36:57.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:57 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:36:58.013 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo ?:2049 2/2 2s ago 15s count:2 2024-09-21T10:36:58.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:58 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:36:58.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:58 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:36:58.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:58 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:36:58.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:58 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:36:58.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:58 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T10:36:58.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:58 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:36:58.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:58 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:36:58.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:58 smithi123 ceph-mon[23709]: pgmap v241: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 2.1 KiB/s wr, 5 op/s 2024-09-21T10:36:58.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:58 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:36:58.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:36:58 smithi123 ceph-mon[23709]: from='client.14590 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:36:58.711 INFO:teuthology.run_tasks:Running task vip.exec... 2024-09-21T10:36:58.729 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi123.front.sepia.ceph.com 2024-09-21T10:36:58.730 DEBUG:teuthology.orchestra.run.smithi123:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'mkdir /mnt/foo' 2024-09-21T10:36:58.756 INFO:teuthology.orchestra.run.smithi123.stderr:+ mkdir /mnt/foo 2024-09-21T10:36:58.764 DEBUG:teuthology.orchestra.run.smithi123:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done' 2024-09-21T10:36:58.828 INFO:teuthology.orchestra.run.smithi123.stderr:++ hostname 2024-09-21T10:36:58.828 INFO:teuthology.orchestra.run.smithi123.stderr:+ mount -t nfs smithi123:/fake /mnt/foo -o sync 2024-09-21T10:36:58.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:58 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:36:58.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:58 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:36:58.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:58 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:36:58.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:58 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:36:58.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:58 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T10:36:58.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:58 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:36:58.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:58 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:36:58.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:58 smithi175 ceph-mon[25806]: pgmap v241: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 2.1 KiB/s wr, 5 op/s 2024-09-21T10:36:58.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:58 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:36:58.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:36:58 smithi175 ceph-mon[25806]: from='client.14590 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:36:59.291 DEBUG:teuthology.orchestra.run.smithi123:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'echo test > /mnt/foo/testfile' 2024-09-21T10:36:59.357 INFO:teuthology.orchestra.run.smithi123.stderr:+ echo test 2024-09-21T10:36:59.367 DEBUG:teuthology.orchestra.run.smithi123:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c sync 2024-09-21T10:36:59.428 INFO:teuthology.orchestra.run.smithi123.stderr:+ sync 2024-09-21T10:36:59.612 INFO:teuthology.run_tasks:Running task parallel... 2024-09-21T10:36:59.622 INFO:teuthology.task.parallel:starting parallel... 2024-09-21T10:36:59.622 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-09-21T10:36:59.623 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-09-21T10:36:59.623 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi123.front.sepia.ceph.com 2024-09-21T10:36:59.623 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -e sha1=5c1cdc3c470dae78e91a0ed2bcf13509a046d301 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-09-21T10:36:59.624 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-09-21T10:36:59.624 INFO:teuthology.task.sequential:In sequential, running task exec... 2024-09-21T10:36:59.625 INFO:teuthology.task.exec:Executing custom commands... 2024-09-21T10:36:59.625 INFO:teuthology.task.exec:Running commands on role host.a host ubuntu@smithi123.front.sepia.ceph.com 2024-09-21T10:36:59.625 DEBUG:teuthology.orchestra.run.smithi123:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'cd /mnt/foo && dbench 5 -t 600 || true' 2024-09-21T10:36:59.682 INFO:teuthology.orchestra.run.smithi123.stdout:dbench version 4.00 - Copyright Andrew Tridgell 1999-2004 2024-09-21T10:36:59.683 INFO:teuthology.orchestra.run.smithi123.stdout: 2024-09-21T10:36:59.683 INFO:teuthology.orchestra.run.smithi123.stdout:Running for 600 seconds with load '/usr/share/dbench/client.txt' and minimum warmup 120 secs 2024-09-21T10:36:59.684 INFO:teuthology.orchestra.run.smithi123.stdout:4 of 5 processes prepared for launch 0 sec 2024-09-21T10:36:59.784 INFO:teuthology.orchestra.run.smithi123.stdout:5 of 5 processes prepared for launch 0 sec 2024-09-21T10:36:59.784 INFO:teuthology.orchestra.run.smithi123.stdout:releasing clients 2024-09-21T10:37:00.784 INFO:teuthology.orchestra.run.smithi123.stdout: 5 175 35.51 MB/sec warmup 1 sec latency 39.024 ms 2024-09-21T10:37:00.954 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:37:00 smithi123 ceph-mon[23709]: pgmap v242: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 2.1 KiB/s wr, 5 op/s 2024-09-21T10:37:01.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:37:00 smithi175 ceph-mon[25806]: pgmap v242: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 2.1 KiB/s wr, 5 op/s 2024-09-21T10:37:01.784 INFO:teuthology.orchestra.run.smithi123.stdout: 5 459 40.29 MB/sec warmup 2 sec latency 43.576 ms 2024-09-21T10:37:02.751 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:37:02 smithi123 ceph-mon[23709]: pgmap v243: 97 pgs: 97 active+clean; 10 MiB data, 52 MiB used, 715 GiB / 715 GiB avail; 2.6 KiB/s rd, 855 KiB/s wr, 20 op/s 2024-09-21T10:37:02.784 INFO:teuthology.orchestra.run.smithi123.stdout: 5 927 36.27 MB/sec warmup 3 sec latency 22.764 ms 2024-09-21T10:37:03.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:37:02 smithi175 ceph-mon[25806]: pgmap v243: 97 pgs: 97 active+clean; 10 MiB data, 52 MiB used, 715 GiB / 715 GiB avail; 2.6 KiB/s rd, 855 KiB/s wr, 20 op/s 2024-09-21T10:37:03.411 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -e sha1=5c1cdc3c470dae78e91a0ed2bcf13509a046d301 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-09-21T10:37:03.784 INFO:teuthology.orchestra.run.smithi123.stdout: 5 1537 32.61 MB/sec warmup 4 sec latency 22.319 ms 2024-09-21T10:37:04.774 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:37:04 smithi123 ceph-mon[23709]: pgmap v244: 97 pgs: 97 active+clean; 29 MiB data, 61 MiB used, 715 GiB / 715 GiB avail; 2.6 KiB/s rd, 2.4 MiB/s wr, 52 op/s 2024-09-21T10:37:04.784 INFO:teuthology.orchestra.run.smithi123.stdout: 5 2388 27.98 MB/sec warmup 5 sec latency 40.757 ms 2024-09-21T10:37:05.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:37:04 smithi175 ceph-mon[25806]: pgmap v244: 97 pgs: 97 active+clean; 29 MiB data, 61 MiB used, 715 GiB / 715 GiB avail; 2.6 KiB/s rd, 2.4 MiB/s wr, 52 op/s 2024-09-21T10:37:05.784 INFO:teuthology.orchestra.run.smithi123.stdout: 5 3443 30.53 MB/sec warmup 6 sec latency 21.756 ms 2024-09-21T10:37:06.785 INFO:teuthology.orchestra.run.smithi123.stdout: 5 3823 28.40 MB/sec warmup 7 sec latency 33.076 ms 2024-09-21T10:37:06.792 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:37:06 smithi123 ceph-mon[23709]: pgmap v245: 97 pgs: 97 active+clean; 108 MiB data, 210 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 9.9 MiB/s wr, 525 op/s 2024-09-21T10:37:07.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:37:06 smithi175 ceph-mon[25806]: pgmap v245: 97 pgs: 97 active+clean; 108 MiB data, 210 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 9.9 MiB/s wr, 525 op/s 2024-09-21T10:37:07.785 INFO:teuthology.orchestra.run.smithi123.stdout: 5 4086 25.08 MB/sec warmup 8 sec latency 40.902 ms 2024-09-21T10:37:08.450 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -e sha1=5c1cdc3c470dae78e91a0ed2bcf13509a046d301 -- bash -c 'ceph config set global log_to_journald false --force' 2024-09-21T10:37:08.786 INFO:teuthology.orchestra.run.smithi123.stdout: 5 4658 23.15 MB/sec warmup 9 sec latency 53.931 ms 2024-09-21T10:37:08.829 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:37:08 smithi123 ceph-mon[23709]: pgmap v246: 97 pgs: 97 active+clean; 123 MiB data, 249 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 12 MiB/s wr, 637 op/s 2024-09-21T10:37:09.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:37:08 smithi175 ceph-mon[25806]: pgmap v246: 97 pgs: 97 active+clean; 123 MiB data, 249 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 12 MiB/s wr, 637 op/s 2024-09-21T10:37:09.785 INFO:teuthology.orchestra.run.smithi123.stdout: 5 5312 22.71 MB/sec warmup 10 sec latency 31.101 ms 2024-09-21T10:37:10.733 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:37:10 smithi123 ceph-mon[23709]: pgmap v247: 97 pgs: 97 active+clean; 126 MiB data, 234 MiB used, 715 GiB / 715 GiB avail; 2.9 KiB/s rd, 12 MiB/s wr, 658 op/s 2024-09-21T10:37:10.785 INFO:teuthology.orchestra.run.smithi123.stdout: 5 6135 22.09 MB/sec warmup 11 sec latency 21.680 ms 2024-09-21T10:37:11.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:37:10 smithi175 ceph-mon[25806]: pgmap v247: 97 pgs: 97 active+clean; 126 MiB data, 234 MiB used, 715 GiB / 715 GiB avail; 2.9 KiB/s rd, 12 MiB/s wr, 658 op/s 2024-09-21T10:37:11.785 INFO:teuthology.orchestra.run.smithi123.stdout: 5 6859 22.73 MB/sec warmup 12 sec latency 37.288 ms 2024-09-21T10:37:12.785 INFO:teuthology.orchestra.run.smithi123.stdout: 5 7223 22.21 MB/sec warmup 13 sec latency 47.385 ms 2024-09-21T10:37:12.887 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:37:12 smithi123 ceph-mon[23709]: pgmap v248: 97 pgs: 97 active+clean; 131 MiB data, 262 MiB used, 715 GiB / 715 GiB avail; 2.9 KiB/s rd, 14 MiB/s wr, 860 op/s 2024-09-21T10:37:13.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:37:12 smithi175 ceph-mon[25806]: pgmap v248: 97 pgs: 97 active+clean; 131 MiB data, 262 MiB used, 715 GiB / 715 GiB avail; 2.9 KiB/s rd, 14 MiB/s wr, 860 op/s 2024-09-21T10:37:13.405 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -e sha1=5c1cdc3c470dae78e91a0ed2bcf13509a046d301 -- bash -c 'ceph mgr module enable nfs --force' 2024-09-21T10:37:13.785 INFO:teuthology.orchestra.run.smithi123.stdout: 5 7785 21.25 MB/sec warmup 14 sec latency 59.137 ms 2024-09-21T10:37:14.785 INFO:teuthology.orchestra.run.smithi123.stdout: 5 8693 21.27 MB/sec warmup 15 sec latency 14.662 ms 2024-09-21T10:37:14.907 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:37:14 smithi123 ceph-mon[23709]: pgmap v249: 97 pgs: 97 active+clean; 131 MiB data, 274 MiB used, 715 GiB / 715 GiB avail; 8.0 KiB/s rd, 14 MiB/s wr, 878 op/s 2024-09-21T10:37:15.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:37:14 smithi175 ceph-mon[25806]: pgmap v249: 97 pgs: 97 active+clean; 131 MiB data, 274 MiB used, 715 GiB / 715 GiB avail; 8.0 KiB/s rd, 14 MiB/s wr, 878 op/s 2024-09-21T10:37:15.785 INFO:teuthology.orchestra.run.smithi123.stdout: 5 9693 21.16 MB/sec warmup 16 sec latency 18.610 ms 2024-09-21T10:37:16.785 INFO:teuthology.orchestra.run.smithi123.stdout: 5 10736 22.49 MB/sec warmup 17 sec latency 19.154 ms 2024-09-21T10:37:16.862 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:37:16 smithi123 ceph-mon[23709]: pgmap v250: 97 pgs: 97 active+clean; 148 MiB data, 343 MiB used, 715 GiB / 715 GiB avail; 8.2 KiB/s rd, 16 MiB/s wr, 1.19k op/s 2024-09-21T10:37:17.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:37:16 smithi175 ceph-mon[25806]: pgmap v250: 97 pgs: 97 active+clean; 148 MiB data, 343 MiB used, 715 GiB / 715 GiB avail; 8.2 KiB/s rd, 16 MiB/s wr, 1.19k op/s 2024-09-21T10:37:17.785 INFO:teuthology.orchestra.run.smithi123.stdout: 5 11307 21.82 MB/sec warmup 18 sec latency 18.448 ms 2024-09-21T10:37:17.830 INFO:teuthology.orchestra.run.smithi123.stderr:module 'nfs' is already enabled 2024-09-21T10:37:17.871 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:37:17 smithi123 ceph-mon[23709]: from='client.? 172.21.15.123:0/2732589354' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-09-21T10:37:18.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:37:17 smithi175 ceph-mon[25806]: from='client.? 172.21.15.123:0/2732589354' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-09-21T10:37:18.785 INFO:teuthology.orchestra.run.smithi123.stdout: 5 12309 21.99 MB/sec warmup 19 sec latency 13.965 ms 2024-09-21T10:37:18.875 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:37:18 smithi123 ceph-mon[23709]: pgmap v251: 97 pgs: 97 active+clean; 152 MiB data, 381 MiB used, 715 GiB / 715 GiB avail; 8.0 KiB/s rd, 9.7 MiB/s wr, 823 op/s 2024-09-21T10:37:18.875 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:37:18 smithi123 ceph-mon[23709]: from='client.? 172.21.15.123:0/2732589354' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "nfs", "force": true}]': finished 2024-09-21T10:37:18.875 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:37:18 smithi123 ceph-mon[23709]: mgrmap e21: smithi123.stnuit(active, since 6m), standbys: smithi175.txvwrq 2024-09-21T10:37:19.058 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -e sha1=5c1cdc3c470dae78e91a0ed2bcf13509a046d301 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-09-21T10:37:19.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:37:18 smithi175 ceph-mon[25806]: pgmap v251: 97 pgs: 97 active+clean; 152 MiB data, 381 MiB used, 715 GiB / 715 GiB avail; 8.0 KiB/s rd, 9.7 MiB/s wr, 823 op/s 2024-09-21T10:37:19.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:37:18 smithi175 ceph-mon[25806]: from='client.? 172.21.15.123:0/2732589354' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "nfs", "force": true}]': finished 2024-09-21T10:37:19.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:37:18 smithi175 ceph-mon[25806]: mgrmap e21: smithi123.stnuit(active, since 6m), standbys: smithi175.txvwrq 2024-09-21T10:37:19.785 INFO:teuthology.orchestra.run.smithi123.stdout: 5 13240 21.71 MB/sec warmup 20 sec latency 17.843 ms 2024-09-21T10:37:20.785 INFO:teuthology.orchestra.run.smithi123.stdout: 5 14209 22.61 MB/sec warmup 21 sec latency 31.709 ms 2024-09-21T10:37:20.886 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:37:20 smithi123 ceph-mon[23709]: pgmap v252: 97 pgs: 97 active+clean; 151 MiB data, 384 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 8.5 MiB/s wr, 743 op/s 2024-09-21T10:37:21.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:37:20 smithi175 ceph-mon[25806]: pgmap v252: 97 pgs: 97 active+clean; 151 MiB data, 384 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 8.5 MiB/s wr, 743 op/s 2024-09-21T10:37:21.785 INFO:teuthology.orchestra.run.smithi123.stdout: 5 14844 22.19 MB/sec warmup 22 sec latency 28.169 ms 2024-09-21T10:37:22.785 INFO:teuthology.orchestra.run.smithi123.stdout: 5 15794 22.31 MB/sec warmup 23 sec latency 12.784 ms 2024-09-21T10:37:22.894 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:37:22 smithi123 ceph-mon[23709]: pgmap v253: 97 pgs: 97 active+clean; 158 MiB data, 450 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 11 MiB/s wr, 1.07k op/s 2024-09-21T10:37:23.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:37:22 smithi175 ceph-mon[25806]: pgmap v253: 97 pgs: 97 active+clean; 158 MiB data, 450 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 11 MiB/s wr, 1.07k op/s 2024-09-21T10:37:23.786 INFO:teuthology.orchestra.run.smithi123.stdout: 5 16787 22.08 MB/sec warmup 24 sec latency 23.854 ms 2024-09-21T10:37:23.851 INFO:teuthology.orchestra.run.smithi123.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:5c1cdc3c470dae78e91a0ed2bcf13509a046d301 2024-09-21T10:37:24.786 INFO:teuthology.orchestra.run.smithi123.stdout: 5 17756 22.81 MB/sec warmup 25 sec latency 26.199 ms 2024-09-21T10:37:24.904 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:37:24 smithi123 ceph-mon[23709]: pgmap v254: 97 pgs: 97 active+clean; 162 MiB data, 489 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 11 MiB/s wr, 987 op/s 2024-09-21T10:37:24.904 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:37:24 smithi123 ceph-mon[23709]: from='client.14610 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:5c1cdc3c470dae78e91a0ed2bcf13509a046d301", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:37:24.904 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:37:24 smithi123 ceph-mon[23709]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:5c1cdc3c470dae78e91a0ed2bcf13509a046d301 2024-09-21T10:37:24.904 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:37:24 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:37:24.905 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:37:24 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:37:24.905 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:37:24 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:37:24.905 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:37:24 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T10:37:24.905 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:37:24 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:37:24.905 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:37:24 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:37:25.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:37:24 smithi175 ceph-mon[25806]: pgmap v254: 97 pgs: 97 active+clean; 162 MiB data, 489 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 11 MiB/s wr, 987 op/s 2024-09-21T10:37:25.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:37:24 smithi175 ceph-mon[25806]: from='client.14610 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:5c1cdc3c470dae78e91a0ed2bcf13509a046d301", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:37:25.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:37:24 smithi175 ceph-mon[25806]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:5c1cdc3c470dae78e91a0ed2bcf13509a046d301 2024-09-21T10:37:25.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:37:24 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:37:25.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:37:24 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:37:25.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:37:24 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:37:25.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:37:24 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T10:37:25.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:37:24 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:37:25.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:37:24 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:37:25.174 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-09-21T10:37:25.174 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi123.front.sepia.ceph.com 2024-09-21T10:37:25.175 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -e sha1=5c1cdc3c470dae78e91a0ed2bcf13509a046d301 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done' 2024-09-21T10:37:25.786 INFO:teuthology.orchestra.run.smithi123.stdout: 5 18378 22.45 MB/sec warmup 26 sec latency 15.452 ms 2024-09-21T10:37:25.910 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:37:25 smithi123 ceph-mon[23709]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:5c1cdc3c470dae78e91a0ed2bcf13509a046d301 2024-09-21T10:37:26.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:37:25 smithi175 ceph-mon[25806]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:5c1cdc3c470dae78e91a0ed2bcf13509a046d301 2024-09-21T10:37:26.786 INFO:teuthology.orchestra.run.smithi123.stdout: 5 18978 22.07 MB/sec warmup 27 sec latency 18.181 ms 2024-09-21T10:37:26.915 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:37:26 smithi123 ceph-mon[23709]: pgmap v255: 97 pgs: 97 active+clean; 172 MiB data, 574 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 14 MiB/s wr, 1.30k op/s 2024-09-21T10:37:27.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:37:26 smithi175 ceph-mon[25806]: pgmap v255: 97 pgs: 97 active+clean; 172 MiB data, 574 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 14 MiB/s wr, 1.30k op/s 2024-09-21T10:37:27.786 INFO:teuthology.orchestra.run.smithi123.stdout: 5 19825 21.86 MB/sec warmup 28 sec latency 23.960 ms 2024-09-21T10:37:28.786 INFO:teuthology.orchestra.run.smithi123.stdout: 5 20249 21.45 MB/sec warmup 29 sec latency 32.882 ms 2024-09-21T10:37:28.942 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:37:28 smithi123 ceph-mon[23709]: pgmap v256: 97 pgs: 97 active+clean; 174 MiB data, 598 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 12 MiB/s wr, 1.01k op/s 2024-09-21T10:37:29.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:37:28 smithi175 ceph-mon[25806]: pgmap v256: 97 pgs: 97 active+clean; 174 MiB data, 598 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 12 MiB/s wr, 1.01k op/s 2024-09-21T10:37:29.518 INFO:teuthology.orchestra.run.smithi123.stdout:true 2024-09-21T10:37:29.786 INFO:teuthology.orchestra.run.smithi123.stdout: 5 20624 21.21 MB/sec warmup 30 sec latency 41.354 ms 2024-09-21T10:37:30.462 INFO:teuthology.orchestra.run.smithi123.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T10:37:30.462 INFO:teuthology.orchestra.run.smithi123.stdout:alertmanager.smithi123 smithi123 *:9093,9094 running (5m) 32s ago 7m 22.5M - ba2b418f427c db0d7d5d885a 2024-09-21T10:37:30.463 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi123 smithi123 running (7m) 32s ago 7m 6405k - 17.2.0 e1d6a67b021e c459f87d5b50 2024-09-21T10:37:30.463 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi175 smithi175 running (6m) 34s ago 6m 6405k - 17.2.0 e1d6a67b021e e35e422d5583 2024-09-21T10:37:30.463 INFO:teuthology.orchestra.run.smithi123.stdout:grafana.smithi123 smithi123 *:3000 running (5m) 32s ago 6m 53.9M - 8.3.5 dad864ee21e9 86ee7306dede 2024-09-21T10:37:30.463 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi123.uqoyow smithi123 running (61s) 32s ago 60s 15.9M - 17.2.0 e1d6a67b021e f0a21eaac266 2024-09-21T10:37:30.463 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi175.jrlevw smithi175 running (58s) 34s ago 57s 11.2M - 17.2.0 e1d6a67b021e b21b16f4bc50 2024-09-21T10:37:30.463 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi123.stnuit smithi123 *:9283 running (8m) 32s ago 8m 459M - 17.2.0 e1d6a67b021e 417fa41b561b 2024-09-21T10:37:30.463 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi175.txvwrq smithi175 *:8443,9283 running (6m) 34s ago 6m 411M - 17.2.0 e1d6a67b021e 79b9f39b87a4 2024-09-21T10:37:30.463 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi123 smithi123 running (8m) 32s ago 8m 46.6M 2048M 17.2.0 e1d6a67b021e 6ad9a8bc53b7 2024-09-21T10:37:30.463 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi175 smithi175 running (6m) 34s ago 6m 40.3M 2048M 17.2.0 e1d6a67b021e 753f564c8cf5 2024-09-21T10:37:30.463 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.0.0.smithi123.ddatpr smithi123 *:2049 running (43s) 32s ago 43s 49.1M - 4.0 e1d6a67b021e 36f8cb2281eb 2024-09-21T10:37:30.463 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.1.0.smithi175.wpawer smithi175 *:2049 running (38s) 34s ago 37s 52.9M - 4.0 e1d6a67b021e 2200aeed9200 2024-09-21T10:37:30.463 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi123 smithi123 *:9100 running (6m) 32s ago 6m 19.7M - 1dbe0e931976 875d5b2d6936 2024-09-21T10:37:30.463 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi175 smithi175 *:9100 running (6m) 34s ago 6m 19.7M - 1dbe0e931976 92ddc27e1ca2 2024-09-21T10:37:30.464 INFO:teuthology.orchestra.run.smithi123.stdout:osd.0 smithi123 running (5m) 32s ago 5m 43.0M 3216M 17.2.0 e1d6a67b021e a94edf88c456 2024-09-21T10:37:30.464 INFO:teuthology.orchestra.run.smithi123.stdout:osd.1 smithi123 running (5m) 32s ago 5m 45.5M 3216M 17.2.0 e1d6a67b021e 59aee515b0f8 2024-09-21T10:37:30.464 INFO:teuthology.orchestra.run.smithi123.stdout:osd.2 smithi123 running (4m) 32s ago 4m 40.5M 3216M 17.2.0 e1d6a67b021e 7db93f8ce5b5 2024-09-21T10:37:30.464 INFO:teuthology.orchestra.run.smithi123.stdout:osd.3 smithi123 running (4m) 32s ago 4m 39.8M 3216M 17.2.0 e1d6a67b021e c1672dea7b1f 2024-09-21T10:37:30.464 INFO:teuthology.orchestra.run.smithi123.stdout:osd.4 smithi175 running (3m) 34s ago 3m 39.4M 3984M 17.2.0 e1d6a67b021e 48fe728c259e 2024-09-21T10:37:30.464 INFO:teuthology.orchestra.run.smithi123.stdout:osd.5 smithi175 running (3m) 34s ago 3m 41.2M 3984M 17.2.0 e1d6a67b021e e270d5a21b14 2024-09-21T10:37:30.464 INFO:teuthology.orchestra.run.smithi123.stdout:osd.6 smithi175 running (2m) 34s ago 2m 38.2M 3984M 17.2.0 e1d6a67b021e 9bcbf018eb20 2024-09-21T10:37:30.464 INFO:teuthology.orchestra.run.smithi123.stdout:osd.7 smithi175 running (2m) 34s ago 2m 39.4M 3984M 17.2.0 e1d6a67b021e e9b8f1f90994 2024-09-21T10:37:30.464 INFO:teuthology.orchestra.run.smithi123.stdout:prometheus.smithi123 smithi123 *:9095 running (6m) 32s ago 5m 59.9M - 514e6a882f6e 2beddd4ece2e 2024-09-21T10:37:30.786 INFO:teuthology.orchestra.run.smithi123.stdout: 5 21260 21.38 MB/sec warmup 31 sec latency 33.552 ms 2024-09-21T10:37:31.035 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:37:30 smithi123 ceph-mon[23709]: from='client.14614 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:37:31.036 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:37:30 smithi123 ceph-mon[23709]: pgmap v257: 97 pgs: 97 active+clean; 173 MiB data, 612 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 11 MiB/s wr, 934 op/s 2024-09-21T10:37:31.036 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:37:30 smithi123 ceph-mon[23709]: from='client.14618 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:37:31.073 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-09-21T10:37:31.073 INFO:teuthology.orchestra.run.smithi123.stdout: "mon": { 2024-09-21T10:37:31.074 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T10:37:31.074 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:37:31.074 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr": { 2024-09-21T10:37:31.074 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T10:37:31.074 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:37:31.074 INFO:teuthology.orchestra.run.smithi123.stdout: "osd": { 2024-09-21T10:37:31.074 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-21T10:37:31.074 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:37:31.074 INFO:teuthology.orchestra.run.smithi123.stdout: "mds": { 2024-09-21T10:37:31.074 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T10:37:31.074 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:37:31.074 INFO:teuthology.orchestra.run.smithi123.stdout: "overall": { 2024-09-21T10:37:31.075 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-09-21T10:37:31.075 INFO:teuthology.orchestra.run.smithi123.stdout: } 2024-09-21T10:37:31.075 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-09-21T10:37:31.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:37:30 smithi175 ceph-mon[25806]: from='client.14614 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:37:31.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:37:30 smithi175 ceph-mon[25806]: pgmap v257: 97 pgs: 97 active+clean; 173 MiB data, 612 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 11 MiB/s wr, 934 op/s 2024-09-21T10:37:31.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:37:30 smithi175 ceph-mon[25806]: from='client.14618 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:37:31.606 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-09-21T10:37:31.606 INFO:teuthology.orchestra.run.smithi123.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:5c1cdc3c470dae78e91a0ed2bcf13509a046d301", 2024-09-21T10:37:31.606 INFO:teuthology.orchestra.run.smithi123.stdout: "in_progress": true, 2024-09-21T10:37:31.606 INFO:teuthology.orchestra.run.smithi123.stdout: "services_complete": [], 2024-09-21T10:37:31.606 INFO:teuthology.orchestra.run.smithi123.stdout: "progress": "", 2024-09-21T10:37:31.606 INFO:teuthology.orchestra.run.smithi123.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:5c1cdc3c470dae78e91a0ed2bcf13509a046d301 image" 2024-09-21T10:37:31.607 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-09-21T10:37:31.786 INFO:teuthology.orchestra.run.smithi123.stdout: 5 21739 21.19 MB/sec warmup 32 sec latency 28.357 ms 2024-09-21T10:37:32.041 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:37:31 smithi123 ceph-mon[23709]: from='client.14622 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:37:32.041 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:37:31 smithi123 ceph-mon[23709]: from='client.? 172.21.15.123:0/2015224495' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T10:37:32.246 INFO:teuthology.orchestra.run.smithi123.stdout:HEALTH_OK 2024-09-21T10:37:32.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:37:31 smithi175 ceph-mon[25806]: from='client.14622 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:37:32.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:37:31 smithi175 ceph-mon[25806]: from='client.? 172.21.15.123:0/2015224495' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T10:37:32.786 INFO:teuthology.orchestra.run.smithi123.stdout: 5 22141 20.66 MB/sec warmup 33 sec latency 30.529 ms 2024-09-21T10:37:33.073 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:37:32 smithi123 ceph-mon[23709]: from='client.24365 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:37:33.073 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:37:32 smithi123 ceph-mon[23709]: pgmap v258: 97 pgs: 97 active+clean; 181 MiB data, 664 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 12 MiB/s wr, 1.19k op/s 2024-09-21T10:37:33.073 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:37:32 smithi123 ceph-mon[23709]: from='client.? 172.21.15.123:0/1103929988' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T10:37:33.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:37:32 smithi175 ceph-mon[25806]: from='client.24365 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:37:33.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:37:32 smithi175 ceph-mon[25806]: pgmap v258: 97 pgs: 97 active+clean; 181 MiB data, 664 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 12 MiB/s wr, 1.19k op/s 2024-09-21T10:37:33.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:37:32 smithi175 ceph-mon[25806]: from='client.? 172.21.15.123:0/1103929988' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T10:37:33.786 INFO:teuthology.orchestra.run.smithi123.stdout: 5 22468 20.31 MB/sec warmup 34 sec latency 119.034 ms 2024-09-21T10:37:34.786 INFO:teuthology.orchestra.run.smithi123.stdout: 5 23087 20.15 MB/sec warmup 35 sec latency 45.832 ms 2024-09-21T10:37:34.930 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:37:34 smithi123 ceph-mon[23709]: pgmap v259: 97 pgs: 97 active+clean; 183 MiB data, 658 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 9.7 MiB/s wr, 923 op/s 2024-09-21T10:37:35.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:37:34 smithi175 ceph-mon[25806]: pgmap v259: 97 pgs: 97 active+clean; 183 MiB data, 658 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 9.7 MiB/s wr, 923 op/s 2024-09-21T10:37:35.786 INFO:teuthology.orchestra.run.smithi123.stdout: 5 23546 19.71 MB/sec warmup 36 sec latency 78.827 ms 2024-09-21T10:37:36.786 INFO:teuthology.orchestra.run.smithi123.stdout: 5 23843 19.41 MB/sec warmup 37 sec latency 73.859 ms 2024-09-21T10:37:37.020 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:37:36 smithi123 ceph-mon[23709]: pgmap v260: 97 pgs: 97 active+clean; 186 MiB data, 698 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 10 MiB/s wr, 970 op/s 2024-09-21T10:37:37.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:37:36 smithi175 ceph-mon[25806]: pgmap v260: 97 pgs: 97 active+clean; 186 MiB data, 698 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 10 MiB/s wr, 970 op/s 2024-09-21T10:37:37.786 INFO:teuthology.orchestra.run.smithi123.stdout: 5 24329 19.52 MB/sec warmup 38 sec latency 40.009 ms 2024-09-21T10:37:38.786 INFO:teuthology.orchestra.run.smithi123.stdout: 5 24783 19.42 MB/sec warmup 39 sec latency 45.940 ms 2024-09-21T10:37:39.044 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:37:38 smithi123 ceph-mon[23709]: pgmap v261: 97 pgs: 97 active+clean; 184 MiB data, 711 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 7.3 MiB/s wr, 662 op/s 2024-09-21T10:37:39.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:37:38 smithi175 ceph-mon[25806]: pgmap v261: 97 pgs: 97 active+clean; 184 MiB data, 711 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 7.3 MiB/s wr, 662 op/s 2024-09-21T10:37:39.786 INFO:teuthology.orchestra.run.smithi123.stdout: 5 25097 19.28 MB/sec warmup 40 sec latency 59.182 ms 2024-09-21T10:37:40.786 INFO:teuthology.orchestra.run.smithi123.stdout: 5 25215 18.86 MB/sec warmup 41 sec latency 115.890 ms 2024-09-21T10:37:40.895 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:37:40 smithi123 ceph-mon[23709]: pgmap v262: 97 pgs: 97 active+clean; 185 MiB data, 735 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 6.6 MiB/s wr, 628 op/s 2024-09-21T10:37:41.345 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:37:40 smithi175 ceph-mon[25806]: pgmap v262: 97 pgs: 97 active+clean; 185 MiB data, 735 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 6.6 MiB/s wr, 628 op/s 2024-09-21T10:37:41.786 INFO:teuthology.orchestra.run.smithi123.stdout: 5 25388 18.44 MB/sec warmup 42 sec latency 197.027 ms 2024-09-21T10:37:42.786 INFO:teuthology.orchestra.run.smithi123.stdout: 5 25734 18.10 MB/sec warmup 43 sec latency 66.572 ms 2024-09-21T10:37:42.965 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:37:42 smithi123 ceph-mon[23709]: pgmap v263: 97 pgs: 97 active+clean; 199 MiB data, 778 MiB used, 714 GiB / 715 GiB avail; 7.3 KiB/s rd, 8.2 MiB/s wr, 804 op/s 2024-09-21T10:37:43.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:37:42 smithi175 ceph-mon[25806]: pgmap v263: 97 pgs: 97 active+clean; 199 MiB data, 778 MiB used, 714 GiB / 715 GiB avail; 7.3 KiB/s rd, 8.2 MiB/s wr, 804 op/s 2024-09-21T10:37:43.786 INFO:teuthology.orchestra.run.smithi123.stdout: 5 26275 18.04 MB/sec warmup 44 sec latency 25.837 ms 2024-09-21T10:37:44.786 INFO:teuthology.orchestra.run.smithi123.stdout: 5 26952 17.86 MB/sec warmup 45 sec latency 33.302 ms 2024-09-21T10:37:44.999 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:37:44 smithi123 ceph-mon[23709]: pgmap v264: 97 pgs: 97 active+clean; 200 MiB data, 801 MiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 5.9 MiB/s wr, 527 op/s 2024-09-21T10:37:45.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:37:44 smithi175 ceph-mon[25806]: pgmap v264: 97 pgs: 97 active+clean; 200 MiB data, 801 MiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 5.9 MiB/s wr, 527 op/s 2024-09-21T10:37:45.786 INFO:teuthology.orchestra.run.smithi123.stdout: 5 27496 17.78 MB/sec warmup 46 sec latency 59.709 ms 2024-09-21T10:37:46.782 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:37:46 smithi123 ceph-mon[23709]: pgmap v265: 97 pgs: 97 active+clean; 195 MiB data, 843 MiB used, 714 GiB / 715 GiB avail; 7.4 KiB/s rd, 6.2 MiB/s wr, 586 op/s 2024-09-21T10:37:46.786 INFO:teuthology.orchestra.run.smithi123.stdout: 5 28255 18.07 MB/sec warmup 47 sec latency 39.891 ms 2024-09-21T10:37:47.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:37:46 smithi175 ceph-mon[25806]: pgmap v265: 97 pgs: 97 active+clean; 195 MiB data, 843 MiB used, 714 GiB / 715 GiB avail; 7.4 KiB/s rd, 6.2 MiB/s wr, 586 op/s 2024-09-21T10:37:47.786 INFO:teuthology.orchestra.run.smithi123.stdout: 5 28791 18.11 MB/sec warmup 48 sec latency 32.898 ms 2024-09-21T10:37:47.801 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:37:47 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.0", "id": [7, 2]}]: dispatch 2024-09-21T10:37:47.801 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:37:47 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2024-09-21T10:37:47.801 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:37:47 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2024-09-21T10:37:47.801 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:37:47 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2024-09-21T10:37:47.801 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:37:47 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.stnuit/trash_purge_schedule"}]: dispatch 2024-09-21T10:37:47.801 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:37:47 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.stnuit/mirror_snapshot_schedule"}]: dispatch 2024-09-21T10:37:48.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:37:47 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.0", "id": [7, 2]}]: dispatch 2024-09-21T10:37:48.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:37:47 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2024-09-21T10:37:48.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:37:47 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2024-09-21T10:37:48.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:37:47 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2024-09-21T10:37:48.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:37:47 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.stnuit/trash_purge_schedule"}]: dispatch 2024-09-21T10:37:48.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:37:47 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.stnuit/mirror_snapshot_schedule"}]: dispatch 2024-09-21T10:37:48.786 INFO:teuthology.orchestra.run.smithi123.stdout: 5 29341 17.84 MB/sec warmup 49 sec latency 52.259 ms 2024-09-21T10:37:48.812 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:37:48 smithi123 ceph-mon[23709]: pgmap v266: 97 pgs: 97 active+clean; 199 MiB data, 832 MiB used, 714 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.6 MiB/s wr, 510 op/s 2024-09-21T10:37:48.813 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:37:48 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.0", "id": [7, 2]}]': finished 2024-09-21T10:37:48.813 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:37:48 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]': finished 2024-09-21T10:37:48.813 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:37:48 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]': finished 2024-09-21T10:37:48.813 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:37:48 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]': finished 2024-09-21T10:37:48.813 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:37:48 smithi123 ceph-mon[23709]: osdmap e69: 8 total, 8 up, 8 in 2024-09-21T10:37:49.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:37:48 smithi175 ceph-mon[25806]: pgmap v266: 97 pgs: 97 active+clean; 199 MiB data, 832 MiB used, 714 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.6 MiB/s wr, 510 op/s 2024-09-21T10:37:49.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:37:48 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.0", "id": [7, 2]}]': finished 2024-09-21T10:37:49.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:37:48 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]': finished 2024-09-21T10:37:49.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:37:48 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]': finished 2024-09-21T10:37:49.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:37:48 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]': finished 2024-09-21T10:37:49.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:37:48 smithi175 ceph-mon[25806]: osdmap e69: 8 total, 8 up, 8 in 2024-09-21T10:37:49.788 INFO:teuthology.orchestra.run.smithi123.stdout: 5 29730 17.73 MB/sec warmup 50 sec latency 53.539 ms 2024-09-21T10:37:49.829 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:37:49 smithi123 ceph-mon[23709]: osdmap e70: 8 total, 8 up, 8 in 2024-09-21T10:37:50.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:37:49 smithi175 ceph-mon[25806]: osdmap e70: 8 total, 8 up, 8 in 2024-09-21T10:37:50.787 INFO:teuthology.orchestra.run.smithi123.stdout: 5 30052 17.57 MB/sec warmup 51 sec latency 76.964 ms 2024-09-21T10:37:50.865 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:37:50 smithi123 ceph-mon[23709]: pgmap v269: 97 pgs: 97 active+clean; 202 MiB data, 848 MiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 4.6 MiB/s wr, 407 op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-09-21T10:37:50.865 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:37:50 smithi123 ceph-mon[23709]: osdmap e71: 8 total, 8 up, 8 in 2024-09-21T10:37:50.865 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:37:50 smithi123 ceph-mon[23709]: osdmap e72: 8 total, 8 up, 8 in 2024-09-21T10:37:51.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:37:50 smithi175 ceph-mon[25806]: pgmap v269: 97 pgs: 97 active+clean; 202 MiB data, 848 MiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 4.6 MiB/s wr, 407 op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-09-21T10:37:51.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:37:50 smithi175 ceph-mon[25806]: osdmap e71: 8 total, 8 up, 8 in 2024-09-21T10:37:51.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:37:50 smithi175 ceph-mon[25806]: osdmap e72: 8 total, 8 up, 8 in 2024-09-21T10:37:51.787 INFO:teuthology.orchestra.run.smithi123.stdout: 5 30592 17.31 MB/sec warmup 52 sec latency 52.140 ms 2024-09-21T10:37:52.787 INFO:teuthology.orchestra.run.smithi123.stdout: 5 31015 17.26 MB/sec warmup 53 sec latency 46.891 ms 2024-09-21T10:37:52.895 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:37:52 smithi123 ceph-mon[23709]: pgmap v272: 97 pgs: 97 active+clean; 212 MiB data, 896 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.3 MiB/s wr, 910 op/s; 3 B/s, 0 keys/s, 0 objects/s recovering 2024-09-21T10:37:52.895 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:37:52 smithi123 ceph-mon[23709]: osdmap e73: 8 total, 8 up, 8 in 2024-09-21T10:37:53.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:37:52 smithi175 ceph-mon[25806]: pgmap v272: 97 pgs: 97 active+clean; 212 MiB data, 896 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.3 MiB/s wr, 910 op/s; 3 B/s, 0 keys/s, 0 objects/s recovering 2024-09-21T10:37:53.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:37:52 smithi175 ceph-mon[25806]: osdmap e73: 8 total, 8 up, 8 in 2024-09-21T10:37:53.787 INFO:teuthology.orchestra.run.smithi123.stdout: 5 31626 17.37 MB/sec warmup 54 sec latency 31.180 ms 2024-09-21T10:37:54.669 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:37:54 smithi123 ceph-mon[23709]: pgmap v274: 97 pgs: 97 active+clean; 210 MiB data, 910 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 8.1 MiB/s wr, 779 op/s; 1.6 MiB/s, 0 keys/s, 0 objects/s recovering 2024-09-21T10:37:54.787 INFO:teuthology.orchestra.run.smithi123.stdout: 5 32126 17.47 MB/sec warmup 55 sec latency 37.878 ms 2024-09-21T10:37:55.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:37:54 smithi175 ceph-mon[25806]: pgmap v274: 97 pgs: 97 active+clean; 210 MiB data, 910 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 8.1 MiB/s wr, 779 op/s; 1.6 MiB/s, 0 keys/s, 0 objects/s recovering 2024-09-21T10:37:55.787 INFO:teuthology.orchestra.run.smithi123.stdout: 5 32369 17.23 MB/sec warmup 56 sec latency 44.426 ms 2024-09-21T10:37:56.787 INFO:teuthology.orchestra.run.smithi123.stdout: 5 32796 16.99 MB/sec warmup 57 sec latency 73.516 ms 2024-09-21T10:37:56.911 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:37:56 smithi123 ceph-mon[23709]: pgmap v275: 97 pgs: 97 active+clean; 220 MiB data, 962 MiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 9.9 MiB/s wr, 1.03k op/s; 1.3 MiB/s, 0 objects/s recovering 2024-09-21T10:37:57.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:37:56 smithi175 ceph-mon[25806]: pgmap v275: 97 pgs: 97 active+clean; 220 MiB data, 962 MiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 9.9 MiB/s wr, 1.03k op/s; 1.3 MiB/s, 0 objects/s recovering 2024-09-21T10:37:57.787 INFO:teuthology.orchestra.run.smithi123.stdout: 5 33072 16.81 MB/sec warmup 58 sec latency 80.156 ms 2024-09-21T10:37:58.699 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:37:58 smithi123 ceph-mon[23709]: pgmap v276: 97 pgs: 97 active+clean; 223 MiB data, 978 MiB used, 714 GiB / 715 GiB avail; 7.5 KiB/s rd, 8.5 MiB/s wr, 815 op/s; 1.0 MiB/s, 0 objects/s recovering 2024-09-21T10:37:58.787 INFO:teuthology.orchestra.run.smithi123.stdout: 5 33573 16.80 MB/sec warmup 59 sec latency 36.125 ms 2024-09-21T10:37:59.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:37:58 smithi175 ceph-mon[25806]: pgmap v276: 97 pgs: 97 active+clean; 223 MiB data, 978 MiB used, 714 GiB / 715 GiB avail; 7.5 KiB/s rd, 8.5 MiB/s wr, 815 op/s; 1.0 MiB/s, 0 objects/s recovering 2024-09-21T10:37:59.787 INFO:teuthology.orchestra.run.smithi123.stdout: 5 34288 16.64 MB/sec warmup 60 sec latency 68.243 ms 2024-09-21T10:38:00.767 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:38:00 smithi123 ceph-mon[23709]: pgmap v277: 97 pgs: 97 active+clean; 222 MiB data, 979 MiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 4.9 MiB/s wr, 419 op/s; 910 KiB/s, 0 objects/s recovering 2024-09-21T10:38:00.767 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:38:00 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:38:00.767 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:38:00 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:38:00.767 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:38:00 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:38:00.787 INFO:teuthology.orchestra.run.smithi123.stdout: 5 34894 16.78 MB/sec warmup 61 sec latency 46.872 ms 2024-09-21T10:38:00.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:38:00 smithi175 ceph-mon[25806]: pgmap v277: 97 pgs: 97 active+clean; 222 MiB data, 979 MiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 4.9 MiB/s wr, 419 op/s; 910 KiB/s, 0 objects/s recovering 2024-09-21T10:38:00.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:38:00 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:38:00.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:38:00 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:38:00.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:38:00 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:38:01.783 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:38:01 smithi123 ceph-mon[23709]: Upgrade: Target is version 19.3.0-5103-g5c1cdc3c (unknown) 2024-09-21T10:38:01.783 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:38:01 smithi123 ceph-mon[23709]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, digests ['quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472'] 2024-09-21T10:38:01.783 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:38:01 smithi123 ceph-mon[23709]: Upgrade: Need to upgrade myself (mgr.smithi123.stnuit) 2024-09-21T10:38:01.783 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:38:01 smithi123 ceph-mon[23709]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472 on smithi175 2024-09-21T10:38:01.787 INFO:teuthology.orchestra.run.smithi123.stdout: 5 35552 16.97 MB/sec warmup 62 sec latency 64.938 ms 2024-09-21T10:38:02.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:38:01 smithi175 ceph-mon[25806]: Upgrade: Target is version 19.3.0-5103-g5c1cdc3c (unknown) 2024-09-21T10:38:02.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:38:01 smithi175 ceph-mon[25806]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, digests ['quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472'] 2024-09-21T10:38:02.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:38:01 smithi175 ceph-mon[25806]: Upgrade: Need to upgrade myself (mgr.smithi123.stnuit) 2024-09-21T10:38:02.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:38:01 smithi175 ceph-mon[25806]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472 on smithi175 2024-09-21T10:38:02.643 INFO:teuthology.orchestra.run.smithi123.stdout:true 2024-09-21T10:38:02.787 INFO:teuthology.orchestra.run.smithi123.stdout: 5 35769 16.80 MB/sec warmup 63 sec latency 80.624 ms 2024-09-21T10:38:02.850 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:38:02 smithi123 ceph-mon[23709]: pgmap v278: 97 pgs: 97 active+clean; 223 MiB data, 1021 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 6.2 MiB/s wr, 617 op/s; 819 KiB/s, 0 objects/s recovering 2024-09-21T10:38:03.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:38:02 smithi175 ceph-mon[25806]: pgmap v278: 97 pgs: 97 active+clean; 223 MiB data, 1021 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 6.2 MiB/s wr, 617 op/s; 819 KiB/s, 0 objects/s recovering 2024-09-21T10:38:03.382 INFO:teuthology.orchestra.run.smithi123.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T10:38:03.382 INFO:teuthology.orchestra.run.smithi123.stdout:alertmanager.smithi123 smithi123 *:9093,9094 running (6m) 65s ago 7m 22.5M - ba2b418f427c db0d7d5d885a 2024-09-21T10:38:03.382 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi123 smithi123 running (7m) 65s ago 7m 6405k - 17.2.0 e1d6a67b021e c459f87d5b50 2024-09-21T10:38:03.382 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi175 smithi175 running (7m) 67s ago 7m 6405k - 17.2.0 e1d6a67b021e e35e422d5583 2024-09-21T10:38:03.382 INFO:teuthology.orchestra.run.smithi123.stdout:grafana.smithi123 smithi123 *:3000 running (6m) 65s ago 7m 53.9M - 8.3.5 dad864ee21e9 86ee7306dede 2024-09-21T10:38:03.382 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi123.uqoyow smithi123 running (93s) 65s ago 93s 15.9M - 17.2.0 e1d6a67b021e f0a21eaac266 2024-09-21T10:38:03.382 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi175.jrlevw smithi175 running (91s) 67s ago 90s 11.2M - 17.2.0 e1d6a67b021e b21b16f4bc50 2024-09-21T10:38:03.382 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi123.stnuit smithi123 *:9283 running (8m) 65s ago 8m 459M - 17.2.0 e1d6a67b021e 417fa41b561b 2024-09-21T10:38:03.382 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi175.txvwrq smithi175 *:8443,9283 running (6m) 67s ago 6m 411M - 17.2.0 e1d6a67b021e 79b9f39b87a4 2024-09-21T10:38:03.382 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi123 smithi123 running (8m) 65s ago 9m 46.6M 2048M 17.2.0 e1d6a67b021e 6ad9a8bc53b7 2024-09-21T10:38:03.383 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi175 smithi175 running (6m) 67s ago 6m 40.3M 2048M 17.2.0 e1d6a67b021e 753f564c8cf5 2024-09-21T10:38:03.383 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.0.0.smithi123.ddatpr smithi123 *:2049 running (76s) 65s ago 76s 49.1M - 4.0 e1d6a67b021e 36f8cb2281eb 2024-09-21T10:38:03.383 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.1.0.smithi175.wpawer smithi175 *:2049 running (71s) 67s ago 70s 52.9M - 4.0 e1d6a67b021e 2200aeed9200 2024-09-21T10:38:03.383 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi123 smithi123 *:9100 running (7m) 65s ago 7m 19.7M - 1dbe0e931976 875d5b2d6936 2024-09-21T10:38:03.383 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi175 smithi175 *:9100 running (6m) 67s ago 6m 19.7M - 1dbe0e931976 92ddc27e1ca2 2024-09-21T10:38:03.383 INFO:teuthology.orchestra.run.smithi123.stdout:osd.0 smithi123 running (6m) 65s ago 6m 43.0M 3216M 17.2.0 e1d6a67b021e a94edf88c456 2024-09-21T10:38:03.383 INFO:teuthology.orchestra.run.smithi123.stdout:osd.1 smithi123 running (5m) 65s ago 5m 45.5M 3216M 17.2.0 e1d6a67b021e 59aee515b0f8 2024-09-21T10:38:03.383 INFO:teuthology.orchestra.run.smithi123.stdout:osd.2 smithi123 running (5m) 65s ago 5m 40.5M 3216M 17.2.0 e1d6a67b021e 7db93f8ce5b5 2024-09-21T10:38:03.383 INFO:teuthology.orchestra.run.smithi123.stdout:osd.3 smithi123 running (4m) 65s ago 4m 39.8M 3216M 17.2.0 e1d6a67b021e c1672dea7b1f 2024-09-21T10:38:03.383 INFO:teuthology.orchestra.run.smithi123.stdout:osd.4 smithi175 running (4m) 67s ago 4m 39.4M 3984M 17.2.0 e1d6a67b021e 48fe728c259e 2024-09-21T10:38:03.383 INFO:teuthology.orchestra.run.smithi123.stdout:osd.5 smithi175 running (3m) 67s ago 3m 41.2M 3984M 17.2.0 e1d6a67b021e e270d5a21b14 2024-09-21T10:38:03.383 INFO:teuthology.orchestra.run.smithi123.stdout:osd.6 smithi175 running (3m) 67s ago 3m 38.2M 3984M 17.2.0 e1d6a67b021e 9bcbf018eb20 2024-09-21T10:38:03.383 INFO:teuthology.orchestra.run.smithi123.stdout:osd.7 smithi175 running (3m) 67s ago 3m 39.4M 3984M 17.2.0 e1d6a67b021e e9b8f1f90994 2024-09-21T10:38:03.383 INFO:teuthology.orchestra.run.smithi123.stdout:prometheus.smithi123 smithi123 *:9095 running (6m) 65s ago 6m 59.9M - 514e6a882f6e 2beddd4ece2e 2024-09-21T10:38:03.787 INFO:teuthology.orchestra.run.smithi123.stdout: 5 35949 16.57 MB/sec warmup 64 sec latency 82.078 ms 2024-09-21T10:38:03.817 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-09-21T10:38:03.817 INFO:teuthology.orchestra.run.smithi123.stdout: "mon": { 2024-09-21T10:38:03.817 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T10:38:03.817 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:38:03.817 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr": { 2024-09-21T10:38:03.817 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T10:38:03.818 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:38:03.818 INFO:teuthology.orchestra.run.smithi123.stdout: "osd": { 2024-09-21T10:38:03.818 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-21T10:38:03.818 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:38:03.818 INFO:teuthology.orchestra.run.smithi123.stdout: "mds": { 2024-09-21T10:38:03.818 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T10:38:03.818 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:38:03.818 INFO:teuthology.orchestra.run.smithi123.stdout: "overall": { 2024-09-21T10:38:03.818 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-09-21T10:38:03.818 INFO:teuthology.orchestra.run.smithi123.stdout: } 2024-09-21T10:38:03.818 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-09-21T10:38:03.908 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:38:03 smithi123 ceph-mon[23709]: from='client.14638 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:38:03.908 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:38:03 smithi123 ceph-mon[23709]: from='client.14642 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:38:04.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:38:03 smithi175 ceph-mon[25806]: from='client.14638 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:38:04.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:38:03 smithi175 ceph-mon[25806]: from='client.14642 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:38:04.164 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-09-21T10:38:04.164 INFO:teuthology.orchestra.run.smithi123.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472", 2024-09-21T10:38:04.164 INFO:teuthology.orchestra.run.smithi123.stdout: "in_progress": true, 2024-09-21T10:38:04.164 INFO:teuthology.orchestra.run.smithi123.stdout: "services_complete": [], 2024-09-21T10:38:04.164 INFO:teuthology.orchestra.run.smithi123.stdout: "progress": "0/23 daemons upgraded", 2024-09-21T10:38:04.164 INFO:teuthology.orchestra.run.smithi123.stdout: "message": "Pulling quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472 image on host smithi175" 2024-09-21T10:38:04.164 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-09-21T10:38:04.569 INFO:teuthology.orchestra.run.smithi123.stdout:HEALTH_OK 2024-09-21T10:38:04.787 INFO:teuthology.orchestra.run.smithi123.stdout: 5 36096 16.34 MB/sec warmup 65 sec latency 90.094 ms 2024-09-21T10:38:04.934 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:38:04 smithi123 ceph-mon[23709]: from='client.14646 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:38:04.934 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:38:04 smithi123 ceph-mon[23709]: pgmap v279: 97 pgs: 97 active+clean; 225 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 5.8 MiB/s wr, 543 op/s; 683 KiB/s, 0 objects/s recovering 2024-09-21T10:38:04.934 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:38:04 smithi123 ceph-mon[23709]: from='client.? 172.21.15.123:0/1033913530' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T10:38:04.934 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:38:04 smithi123 ceph-mon[23709]: from='client.14654 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:38:04.934 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:38:04 smithi123 ceph-mon[23709]: from='client.? 172.21.15.123:0/691420706' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T10:38:05.126 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:38:04 smithi175 ceph-mon[25806]: from='client.14646 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:38:05.126 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:38:04 smithi175 ceph-mon[25806]: pgmap v279: 97 pgs: 97 active+clean; 225 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 5.8 MiB/s wr, 543 op/s; 683 KiB/s, 0 objects/s recovering 2024-09-21T10:38:05.127 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:38:04 smithi175 ceph-mon[25806]: from='client.? 172.21.15.123:0/1033913530' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T10:38:05.127 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:38:04 smithi175 ceph-mon[25806]: from='client.14654 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:38:05.127 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:38:04 smithi175 ceph-mon[25806]: from='client.? 172.21.15.123:0/691420706' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T10:38:05.788 INFO:teuthology.orchestra.run.smithi123.stdout: 5 36580 16.23 MB/sec warmup 66 sec latency 96.784 ms 2024-09-21T10:38:06.692 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:38:06 smithi123 ceph-mon[23709]: pgmap v280: 97 pgs: 97 active+clean; 239 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 7.7 KiB/s rd, 7.5 MiB/s wr, 718 op/s 2024-09-21T10:38:06.788 INFO:teuthology.orchestra.run.smithi123.stdout: 5 37338 16.24 MB/sec warmup 67 sec latency 44.456 ms 2024-09-21T10:38:07.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:38:06 smithi175 ceph-mon[25806]: pgmap v280: 97 pgs: 97 active+clean; 239 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 7.7 KiB/s rd, 7.5 MiB/s wr, 718 op/s 2024-09-21T10:38:07.788 INFO:teuthology.orchestra.run.smithi123.stdout: 5 37819 16.08 MB/sec warmup 68 sec latency 52.519 ms 2024-09-21T10:38:08.760 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:38:08 smithi123 ceph-mon[23709]: pgmap v281: 97 pgs: 97 active+clean; 235 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.7 MiB/s wr, 521 op/s 2024-09-21T10:38:08.798 INFO:teuthology.orchestra.run.smithi123.stdout: 5 38225 16.11 MB/sec warmup 69 sec latency 85.460 ms 2024-09-21T10:38:09.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:38:08 smithi175 ceph-mon[25806]: pgmap v281: 97 pgs: 97 active+clean; 235 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.7 MiB/s wr, 521 op/s 2024-09-21T10:38:09.798 INFO:teuthology.orchestra.run.smithi123.stdout: 5 38860 16.27 MB/sec warmup 70 sec latency 33.255 ms 2024-09-21T10:38:10.776 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:38:10 smithi123 ceph-mon[23709]: pgmap v282: 97 pgs: 97 active+clean; 240 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 6.0 MiB/s wr, 612 op/s 2024-09-21T10:38:10.798 INFO:teuthology.orchestra.run.smithi123.stdout: 5 39407 16.27 MB/sec warmup 71 sec latency 25.200 ms 2024-09-21T10:38:11.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:38:10 smithi175 ceph-mon[25806]: pgmap v282: 97 pgs: 97 active+clean; 240 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 6.0 MiB/s wr, 612 op/s 2024-09-21T10:38:11.798 INFO:teuthology.orchestra.run.smithi123.stdout: 5 39812 16.09 MB/sec warmup 72 sec latency 52.688 ms 2024-09-21T10:38:12.798 INFO:teuthology.orchestra.run.smithi123.stdout: 5 40162 15.95 MB/sec warmup 73 sec latency 56.959 ms 2024-09-21T10:38:12.839 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:38:12 smithi123 ceph-mon[23709]: pgmap v283: 97 pgs: 97 active+clean; 244 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 7.2 KiB/s rd, 7.5 MiB/s wr, 756 op/s 2024-09-21T10:38:13.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:38:12 smithi175 ceph-mon[25806]: pgmap v283: 97 pgs: 97 active+clean; 244 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 7.2 KiB/s rd, 7.5 MiB/s wr, 756 op/s 2024-09-21T10:38:13.798 INFO:teuthology.orchestra.run.smithi123.stdout: 5 40695 15.94 MB/sec warmup 74 sec latency 32.530 ms 2024-09-21T10:38:14.798 INFO:teuthology.orchestra.run.smithi123.stdout: 5 41182 15.83 MB/sec warmup 75 sec latency 84.828 ms 2024-09-21T10:38:14.885 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:38:14 smithi123 ceph-mon[23709]: pgmap v284: 97 pgs: 97 active+clean; 250 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 6.6 MiB/s wr, 635 op/s 2024-09-21T10:38:15.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:38:14 smithi175 ceph-mon[25806]: pgmap v284: 97 pgs: 97 active+clean; 250 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 6.6 MiB/s wr, 635 op/s 2024-09-21T10:38:15.798 INFO:teuthology.orchestra.run.smithi123.stdout: 5 41513 15.71 MB/sec warmup 76 sec latency 41.636 ms 2024-09-21T10:38:16.696 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:38:16 smithi123 ceph-mon[23709]: pgmap v285: 97 pgs: 97 active+clean; 247 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 8.8 KiB/s rd, 7.6 MiB/s wr, 787 op/s 2024-09-21T10:38:16.798 INFO:teuthology.orchestra.run.smithi123.stdout: 5 41881 15.72 MB/sec warmup 77 sec latency 57.085 ms 2024-09-21T10:38:17.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:38:16 smithi175 ceph-mon[25806]: pgmap v285: 97 pgs: 97 active+clean; 247 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 8.8 KiB/s rd, 7.6 MiB/s wr, 787 op/s 2024-09-21T10:38:17.798 INFO:teuthology.orchestra.run.smithi123.stdout: 5 42463 15.84 MB/sec warmup 78 sec latency 63.836 ms 2024-09-21T10:38:18.714 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:38:18 smithi123 ceph-mon[23709]: pgmap v286: 97 pgs: 97 active+clean; 246 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.2 MiB/s wr, 622 op/s 2024-09-21T10:38:18.798 INFO:teuthology.orchestra.run.smithi123.stdout: 5 42943 15.83 MB/sec warmup 79 sec latency 54.156 ms 2024-09-21T10:38:19.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:38:18 smithi175 ceph-mon[25806]: pgmap v286: 97 pgs: 97 active+clean; 246 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.2 MiB/s wr, 622 op/s 2024-09-21T10:38:19.798 INFO:teuthology.orchestra.run.smithi123.stdout: 5 43246 15.67 MB/sec warmup 80 sec latency 69.898 ms 2024-09-21T10:38:20.676 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:38:20 smithi123 ceph-mon[23709]: pgmap v287: 97 pgs: 97 active+clean; 251 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 6.5 MiB/s wr, 672 op/s 2024-09-21T10:38:20.798 INFO:teuthology.orchestra.run.smithi123.stdout: 5 43510 15.54 MB/sec warmup 81 sec latency 73.000 ms 2024-09-21T10:38:21.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:38:20 smithi175 ceph-mon[25806]: pgmap v287: 97 pgs: 97 active+clean; 251 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 6.5 MiB/s wr, 672 op/s 2024-09-21T10:38:21.798 INFO:teuthology.orchestra.run.smithi123.stdout: 5 43890 15.48 MB/sec warmup 82 sec latency 32.512 ms 2024-09-21T10:38:22.798 INFO:teuthology.orchestra.run.smithi123.stdout: 5 44400 15.38 MB/sec warmup 83 sec latency 47.724 ms 2024-09-21T10:38:22.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:38:22 smithi123 ceph-mon[23709]: pgmap v288: 97 pgs: 97 active+clean; 260 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 7.2 KiB/s rd, 7.3 MiB/s wr, 675 op/s 2024-09-21T10:38:23.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:38:22 smithi175 ceph-mon[25806]: pgmap v288: 97 pgs: 97 active+clean; 260 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 7.2 KiB/s rd, 7.3 MiB/s wr, 675 op/s 2024-09-21T10:38:23.798 INFO:teuthology.orchestra.run.smithi123.stdout: 5 44752 15.29 MB/sec warmup 84 sec latency 52.851 ms 2024-09-21T10:38:24.712 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:38:24 smithi123 ceph-mon[23709]: pgmap v289: 97 pgs: 97 active+clean; 266 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 6.2 MiB/s wr, 613 op/s 2024-09-21T10:38:24.798 INFO:teuthology.orchestra.run.smithi123.stdout: 5 45120 15.21 MB/sec warmup 85 sec latency 78.007 ms 2024-09-21T10:38:25.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:38:24 smithi175 ceph-mon[25806]: pgmap v289: 97 pgs: 97 active+clean; 266 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 6.2 MiB/s wr, 613 op/s 2024-09-21T10:38:25.799 INFO:teuthology.orchestra.run.smithi123.stdout: 5 45503 15.21 MB/sec warmup 86 sec latency 75.293 ms 2024-09-21T10:38:26.791 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:38:26 smithi123 ceph-mon[23709]: pgmap v290: 97 pgs: 97 active+clean; 263 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 7.6 KiB/s rd, 6.5 MiB/s wr, 668 op/s 2024-09-21T10:38:26.798 INFO:teuthology.orchestra.run.smithi123.stdout: 5 46070 15.33 MB/sec warmup 87 sec latency 79.518 ms 2024-09-21T10:38:27.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:38:26 smithi175 ceph-mon[25806]: pgmap v290: 97 pgs: 97 active+clean; 263 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 7.6 KiB/s rd, 6.5 MiB/s wr, 668 op/s 2024-09-21T10:38:27.799 INFO:teuthology.orchestra.run.smithi123.stdout: 5 46457 15.30 MB/sec warmup 88 sec latency 50.094 ms 2024-09-21T10:38:28.799 INFO:teuthology.orchestra.run.smithi123.stdout: 5 46700 15.15 MB/sec warmup 89 sec latency 100.420 ms 2024-09-21T10:38:28.829 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:38:28 smithi123 ceph-mon[23709]: pgmap v291: 97 pgs: 97 active+clean; 263 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.5 MiB/s wr, 514 op/s 2024-09-21T10:38:29.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:38:28 smithi175 ceph-mon[25806]: pgmap v291: 97 pgs: 97 active+clean; 263 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.5 MiB/s wr, 514 op/s 2024-09-21T10:38:29.799 INFO:teuthology.orchestra.run.smithi123.stdout: 5 47012 15.02 MB/sec warmup 90 sec latency 47.077 ms 2024-09-21T10:38:30.799 INFO:teuthology.orchestra.run.smithi123.stdout: 5 47307 14.93 MB/sec warmup 91 sec latency 86.394 ms 2024-09-21T10:38:30.862 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:38:30 smithi123 ceph-mon[23709]: pgmap v292: 97 pgs: 97 active+clean; 263 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 9.5 KiB/s rd, 5.2 MiB/s wr, 494 op/s 2024-09-21T10:38:31.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:38:30 smithi175 ceph-mon[25806]: pgmap v292: 97 pgs: 97 active+clean; 263 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 9.5 KiB/s rd, 5.2 MiB/s wr, 494 op/s 2024-09-21T10:38:31.799 INFO:teuthology.orchestra.run.smithi123.stdout: 5 47680 14.87 MB/sec warmup 92 sec latency 63.732 ms 2024-09-21T10:38:32.799 INFO:teuthology.orchestra.run.smithi123.stdout: 5 48228 14.83 MB/sec warmup 93 sec latency 72.364 ms 2024-09-21T10:38:32.909 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:38:32 smithi123 ceph-mon[23709]: pgmap v293: 97 pgs: 97 active+clean; 277 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 6.7 KiB/s rd, 6.5 MiB/s wr, 615 op/s 2024-09-21T10:38:33.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:38:32 smithi175 ceph-mon[25806]: pgmap v293: 97 pgs: 97 active+clean; 277 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 6.7 KiB/s rd, 6.5 MiB/s wr, 615 op/s 2024-09-21T10:38:33.799 INFO:teuthology.orchestra.run.smithi123.stdout: 5 48836 14.86 MB/sec warmup 94 sec latency 65.469 ms 2024-09-21T10:38:34.724 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:38:34 smithi123 ceph-mon[23709]: pgmap v294: 97 pgs: 97 active+clean; 275 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 5.2 MiB/s wr, 516 op/s 2024-09-21T10:38:34.799 INFO:teuthology.orchestra.run.smithi123.stdout: 5 48942 14.74 MB/sec warmup 95 sec latency 72.234 ms 2024-09-21T10:38:34.942 INFO:teuthology.orchestra.run.smithi123.stdout:true 2024-09-21T10:38:35.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:38:34 smithi175 ceph-mon[25806]: pgmap v294: 97 pgs: 97 active+clean; 275 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 5.2 MiB/s wr, 516 op/s 2024-09-21T10:38:35.618 INFO:teuthology.orchestra.run.smithi123.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T10:38:35.618 INFO:teuthology.orchestra.run.smithi123.stdout:alertmanager.smithi123 smithi123 *:9093,9094 running (6m) 98s ago 8m 22.5M - ba2b418f427c db0d7d5d885a 2024-09-21T10:38:35.618 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi123 smithi123 running (8m) 98s ago 8m 6405k - 17.2.0 e1d6a67b021e c459f87d5b50 2024-09-21T10:38:35.618 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi175 smithi175 running (7m) 100s ago 7m 6405k - 17.2.0 e1d6a67b021e e35e422d5583 2024-09-21T10:38:35.619 INFO:teuthology.orchestra.run.smithi123.stdout:grafana.smithi123 smithi123 *:3000 running (6m) 98s ago 7m 53.9M - 8.3.5 dad864ee21e9 86ee7306dede 2024-09-21T10:38:35.619 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi123.uqoyow smithi123 running (2m) 98s ago 2m 15.9M - 17.2.0 e1d6a67b021e f0a21eaac266 2024-09-21T10:38:35.619 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi175.jrlevw smithi175 running (2m) 100s ago 2m 11.2M - 17.2.0 e1d6a67b021e b21b16f4bc50 2024-09-21T10:38:35.619 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi123.stnuit smithi123 *:9283 running (9m) 98s ago 9m 459M - 17.2.0 e1d6a67b021e 417fa41b561b 2024-09-21T10:38:35.619 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi175.txvwrq smithi175 *:8443,9283 running (7m) 100s ago 7m 411M - 17.2.0 e1d6a67b021e 79b9f39b87a4 2024-09-21T10:38:35.619 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi123 smithi123 running (9m) 98s ago 9m 46.6M 2048M 17.2.0 e1d6a67b021e 6ad9a8bc53b7 2024-09-21T10:38:35.619 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi175 smithi175 running (7m) 100s ago 7m 40.3M 2048M 17.2.0 e1d6a67b021e 753f564c8cf5 2024-09-21T10:38:35.619 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.0.0.smithi123.ddatpr smithi123 *:2049 running (108s) 98s ago 108s 49.1M - 4.0 e1d6a67b021e 36f8cb2281eb 2024-09-21T10:38:35.619 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.1.0.smithi175.wpawer smithi175 *:2049 running (103s) 100s ago 102s 52.9M - 4.0 e1d6a67b021e 2200aeed9200 2024-09-21T10:38:35.619 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi123 smithi123 *:9100 running (7m) 98s ago 7m 19.7M - 1dbe0e931976 875d5b2d6936 2024-09-21T10:38:35.619 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi175 smithi175 *:9100 running (7m) 100s ago 7m 19.7M - 1dbe0e931976 92ddc27e1ca2 2024-09-21T10:38:35.619 INFO:teuthology.orchestra.run.smithi123.stdout:osd.0 smithi123 running (6m) 98s ago 6m 43.0M 3216M 17.2.0 e1d6a67b021e a94edf88c456 2024-09-21T10:38:35.619 INFO:teuthology.orchestra.run.smithi123.stdout:osd.1 smithi123 running (6m) 98s ago 6m 45.5M 3216M 17.2.0 e1d6a67b021e 59aee515b0f8 2024-09-21T10:38:35.619 INFO:teuthology.orchestra.run.smithi123.stdout:osd.2 smithi123 running (5m) 98s ago 5m 40.5M 3216M 17.2.0 e1d6a67b021e 7db93f8ce5b5 2024-09-21T10:38:35.620 INFO:teuthology.orchestra.run.smithi123.stdout:osd.3 smithi123 running (5m) 98s ago 5m 39.8M 3216M 17.2.0 e1d6a67b021e c1672dea7b1f 2024-09-21T10:38:35.620 INFO:teuthology.orchestra.run.smithi123.stdout:osd.4 smithi175 running (4m) 100s ago 4m 39.4M 3984M 17.2.0 e1d6a67b021e 48fe728c259e 2024-09-21T10:38:35.620 INFO:teuthology.orchestra.run.smithi123.stdout:osd.5 smithi175 running (4m) 100s ago 4m 41.2M 3984M 17.2.0 e1d6a67b021e e270d5a21b14 2024-09-21T10:38:35.620 INFO:teuthology.orchestra.run.smithi123.stdout:osd.6 smithi175 running (4m) 100s ago 4m 38.2M 3984M 17.2.0 e1d6a67b021e 9bcbf018eb20 2024-09-21T10:38:35.620 INFO:teuthology.orchestra.run.smithi123.stdout:osd.7 smithi175 running (3m) 100s ago 3m 39.4M 3984M 17.2.0 e1d6a67b021e e9b8f1f90994 2024-09-21T10:38:35.620 INFO:teuthology.orchestra.run.smithi123.stdout:prometheus.smithi123 smithi123 *:9095 running (7m) 98s ago 7m 59.9M - 514e6a882f6e 2beddd4ece2e 2024-09-21T10:38:35.780 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:38:35 smithi123 ceph-mon[23709]: from='client.14662 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:38:35.780 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:38:35 smithi123 ceph-mon[23709]: from='client.14666 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:38:35.799 INFO:teuthology.orchestra.run.smithi123.stdout: 5 49108 14.63 MB/sec warmup 96 sec latency 86.087 ms 2024-09-21T10:38:36.017 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-09-21T10:38:36.017 INFO:teuthology.orchestra.run.smithi123.stdout: "mon": { 2024-09-21T10:38:36.017 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T10:38:36.017 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:38:36.017 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr": { 2024-09-21T10:38:36.017 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T10:38:36.017 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:38:36.017 INFO:teuthology.orchestra.run.smithi123.stdout: "osd": { 2024-09-21T10:38:36.017 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-21T10:38:36.018 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:38:36.018 INFO:teuthology.orchestra.run.smithi123.stdout: "mds": { 2024-09-21T10:38:36.018 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T10:38:36.018 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:38:36.018 INFO:teuthology.orchestra.run.smithi123.stdout: "overall": { 2024-09-21T10:38:36.018 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-09-21T10:38:36.018 INFO:teuthology.orchestra.run.smithi123.stdout: } 2024-09-21T10:38:36.018 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-09-21T10:38:36.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:38:35 smithi175 ceph-mon[25806]: from='client.14662 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:38:36.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:38:35 smithi175 ceph-mon[25806]: from='client.14666 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:38:36.382 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-09-21T10:38:36.382 INFO:teuthology.orchestra.run.smithi123.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472", 2024-09-21T10:38:36.382 INFO:teuthology.orchestra.run.smithi123.stdout: "in_progress": true, 2024-09-21T10:38:36.382 INFO:teuthology.orchestra.run.smithi123.stdout: "services_complete": [], 2024-09-21T10:38:36.382 INFO:teuthology.orchestra.run.smithi123.stdout: "progress": "0/23 daemons upgraded", 2024-09-21T10:38:36.382 INFO:teuthology.orchestra.run.smithi123.stdout: "message": "Pulling quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472 image on host smithi175" 2024-09-21T10:38:36.382 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-09-21T10:38:36.797 INFO:teuthology.orchestra.run.smithi123.stdout:HEALTH_OK 2024-09-21T10:38:36.799 INFO:teuthology.orchestra.run.smithi123.stdout: 5 49347 14.61 MB/sec warmup 97 sec latency 98.585 ms 2024-09-21T10:38:36.883 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:38:36 smithi123 ceph-mon[23709]: from='client.14670 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:38:36.883 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:38:36 smithi123 ceph-mon[23709]: pgmap v295: 97 pgs: 97 active+clean; 277 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 6.9 KiB/s rd, 6.0 MiB/s wr, 587 op/s 2024-09-21T10:38:36.883 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:38:36 smithi123 ceph-mon[23709]: from='client.? 172.21.15.123:0/2745146721' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T10:38:37.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:38:36 smithi175 ceph-mon[25806]: from='client.14670 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:38:37.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:38:36 smithi175 ceph-mon[25806]: pgmap v295: 97 pgs: 97 active+clean; 277 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 6.9 KiB/s rd, 6.0 MiB/s wr, 587 op/s 2024-09-21T10:38:37.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:38:36 smithi175 ceph-mon[25806]: from='client.? 172.21.15.123:0/2745146721' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T10:38:37.728 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:38:37 smithi123 ceph-mon[23709]: from='client.14678 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:38:37.728 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:38:37 smithi123 ceph-mon[23709]: from='client.? 172.21.15.123:0/1661982912' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T10:38:37.799 INFO:teuthology.orchestra.run.smithi123.stdout: 5 49511 14.55 MB/sec warmup 98 sec latency 109.396 ms 2024-09-21T10:38:38.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:38:37 smithi175 ceph-mon[25806]: from='client.14678 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:38:38.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:38:37 smithi175 ceph-mon[25806]: from='client.? 172.21.15.123:0/1661982912' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T10:38:38.766 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:38:38 smithi123 ceph-mon[23709]: pgmap v296: 97 pgs: 97 active+clean; 278 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.3 MiB/s wr, 458 op/s 2024-09-21T10:38:38.799 INFO:teuthology.orchestra.run.smithi123.stdout: 5 49683 14.45 MB/sec warmup 99 sec latency 201.475 ms 2024-09-21T10:38:39.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:38:38 smithi175 ceph-mon[25806]: pgmap v296: 97 pgs: 97 active+clean; 278 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.3 MiB/s wr, 458 op/s 2024-09-21T10:38:39.799 INFO:teuthology.orchestra.run.smithi123.stdout: 5 49799 14.33 MB/sec warmup 100 sec latency 136.336 ms 2024-09-21T10:38:40.799 INFO:teuthology.orchestra.run.smithi123.stdout: 5 50072 14.27 MB/sec warmup 101 sec latency 191.125 ms 2024-09-21T10:38:40.842 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:38:40 smithi123 ceph-mon[23709]: pgmap v297: 97 pgs: 97 active+clean; 282 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.1 MiB/s wr, 498 op/s 2024-09-21T10:38:41.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:38:40 smithi175 ceph-mon[25806]: pgmap v297: 97 pgs: 97 active+clean; 282 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.1 MiB/s wr, 498 op/s 2024-09-21T10:38:41.799 INFO:teuthology.orchestra.run.smithi123.stdout: 5 50566 14.20 MB/sec warmup 102 sec latency 43.070 ms 2024-09-21T10:38:42.799 INFO:teuthology.orchestra.run.smithi123.stdout: 5 51124 14.20 MB/sec warmup 103 sec latency 37.620 ms 2024-09-21T10:38:42.863 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:38:42 smithi123 ceph-mon[23709]: pgmap v298: 97 pgs: 97 active+clean; 285 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 6.4 KiB/s rd, 5.5 MiB/s wr, 509 op/s 2024-09-21T10:38:42.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:38:42 smithi175 ceph-mon[25806]: pgmap v298: 97 pgs: 97 active+clean; 285 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 6.4 KiB/s rd, 5.5 MiB/s wr, 509 op/s 2024-09-21T10:38:43.799 INFO:teuthology.orchestra.run.smithi123.stdout: 5 51719 14.17 MB/sec warmup 104 sec latency 44.911 ms 2024-09-21T10:38:44.629 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:38:44 smithi123 ceph-mon[23709]: Upgrade: Updating mgr.smithi175.txvwrq 2024-09-21T10:38:44.629 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:38:44 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:38:44.629 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:38:44 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi175.txvwrq", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T10:38:44.629 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:38:44 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T10:38:44.629 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:38:44 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:38:44.630 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:38:44 smithi123 ceph-mon[23709]: Deploying daemon mgr.smithi175.txvwrq on smithi175 2024-09-21T10:38:44.630 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:38:44 smithi123 ceph-mon[23709]: pgmap v299: 97 pgs: 97 active+clean; 287 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 9.6 KiB/s rd, 3.8 MiB/s wr, 352 op/s 2024-09-21T10:38:44.799 INFO:teuthology.orchestra.run.smithi123.stdout: 5 52306 14.20 MB/sec warmup 105 sec latency 38.338 ms 2024-09-21T10:38:44.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:38:44 smithi175 ceph-mon[25806]: Upgrade: Updating mgr.smithi175.txvwrq 2024-09-21T10:38:44.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:38:44 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:38:44.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:38:44 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi175.txvwrq", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T10:38:44.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:38:44 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T10:38:44.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:38:44 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:38:44.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:38:44 smithi175 ceph-mon[25806]: Deploying daemon mgr.smithi175.txvwrq on smithi175 2024-09-21T10:38:44.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:38:44 smithi175 ceph-mon[25806]: pgmap v299: 97 pgs: 97 active+clean; 287 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 9.6 KiB/s rd, 3.8 MiB/s wr, 352 op/s 2024-09-21T10:38:45.799 INFO:teuthology.orchestra.run.smithi123.stdout: 5 52829 14.24 MB/sec warmup 106 sec latency 53.740 ms 2024-09-21T10:38:46.800 INFO:teuthology.orchestra.run.smithi123.stdout: 5 53362 14.32 MB/sec warmup 107 sec latency 43.397 ms 2024-09-21T10:38:46.900 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:38:46 smithi123 ceph-mon[23709]: pgmap v300: 97 pgs: 97 active+clean; 288 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 5.2 KiB/s rd, 5.4 MiB/s wr, 575 op/s 2024-09-21T10:38:47.117 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:38:46 smithi175 ceph-mon[25806]: pgmap v300: 97 pgs: 97 active+clean; 288 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 5.2 KiB/s rd, 5.4 MiB/s wr, 575 op/s 2024-09-21T10:38:47.799 INFO:teuthology.orchestra.run.smithi123.stdout: 5 53722 14.25 MB/sec warmup 108 sec latency 68.969 ms 2024-09-21T10:38:48.465 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:38:48 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:38:48.465 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:38:48 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:38:48.465 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:38:48 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:38:48.465 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:38:48 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T10:38:48.465 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:38:48 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.stnuit/trash_purge_schedule"}]: dispatch 2024-09-21T10:38:48.465 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:38:48 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.stnuit/mirror_snapshot_schedule"}]: dispatch 2024-09-21T10:38:48.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:38:48 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:38:48.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:38:48 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:38:48.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:38:48 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:38:48.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:38:48 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T10:38:48.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:38:48 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.stnuit/trash_purge_schedule"}]: dispatch 2024-09-21T10:38:48.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:38:48 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.stnuit/mirror_snapshot_schedule"}]: dispatch 2024-09-21T10:38:48.799 INFO:teuthology.orchestra.run.smithi123.stdout: 5 54050 14.15 MB/sec warmup 109 sec latency 80.896 ms 2024-09-21T10:38:49.485 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:38:49 smithi123 ceph-mon[23709]: pgmap v301: 97 pgs: 97 active+clean; 291 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 5.1 KiB/s rd, 4.8 MiB/s wr, 424 op/s 2024-09-21T10:38:49.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:38:49 smithi175 ceph-mon[25806]: pgmap v301: 97 pgs: 97 active+clean; 291 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 5.1 KiB/s rd, 4.8 MiB/s wr, 424 op/s 2024-09-21T10:38:49.799 INFO:teuthology.orchestra.run.smithi123.stdout: 5 54841 14.20 MB/sec warmup 110 sec latency 30.212 ms 2024-09-21T10:38:50.514 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:38:50 smithi123 ceph-mon[23709]: pgmap v302: 97 pgs: 97 active+clean; 297 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.3 MiB/s wr, 518 op/s 2024-09-21T10:38:50.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:38:50 smithi175 ceph-mon[25806]: pgmap v302: 97 pgs: 97 active+clean; 297 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.3 MiB/s wr, 518 op/s 2024-09-21T10:38:50.799 INFO:teuthology.orchestra.run.smithi123.stdout: 5 55660 14.21 MB/sec warmup 111 sec latency 33.482 ms 2024-09-21T10:38:51.799 INFO:teuthology.orchestra.run.smithi123.stdout: 5 56180 14.26 MB/sec warmup 112 sec latency 36.750 ms 2024-09-21T10:38:52.556 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:38:52 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:38:52.556 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:38:52 smithi123 ceph-mon[23709]: pgmap v303: 97 pgs: 97 active+clean; 305 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 6.7 KiB/s rd, 6.9 MiB/s wr, 666 op/s 2024-09-21T10:38:52.556 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:38:52 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:38:52.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:38:52 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:38:52.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:38:52 smithi175 ceph-mon[25806]: pgmap v303: 97 pgs: 97 active+clean; 305 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 6.7 KiB/s rd, 6.9 MiB/s wr, 666 op/s 2024-09-21T10:38:52.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:38:52 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:38:52.799 INFO:teuthology.orchestra.run.smithi123.stdout: 5 56553 14.28 MB/sec warmup 113 sec latency 62.605 ms 2024-09-21T10:38:53.799 INFO:teuthology.orchestra.run.smithi123.stdout: 5 56996 14.29 MB/sec warmup 114 sec latency 56.885 ms 2024-09-21T10:38:54.799 INFO:teuthology.orchestra.run.smithi123.stdout: 5 57432 14.22 MB/sec warmup 115 sec latency 32.464 ms 2024-09-21T10:38:54.843 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:38:54 smithi123 ceph-mon[23709]: pgmap v304: 97 pgs: 97 active+clean; 303 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 6.6 MiB/s wr, 670 op/s 2024-09-21T10:38:54.843 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:38:54 smithi123 ceph-mon[23709]: Standby manager daemon smithi175.txvwrq restarted 2024-09-21T10:38:54.843 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:38:54 smithi123 ceph-mon[23709]: Standby manager daemon smithi175.txvwrq started 2024-09-21T10:38:54.843 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:38:54 smithi123 ceph-mon[23709]: from='mgr.? 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi175.txvwrq/crt"}]: dispatch 2024-09-21T10:38:54.843 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:38:54 smithi123 ceph-mon[23709]: from='mgr.? 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T10:38:54.843 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:38:54 smithi123 ceph-mon[23709]: from='mgr.? 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi175.txvwrq/key"}]: dispatch 2024-09-21T10:38:54.843 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:38:54 smithi123 ceph-mon[23709]: from='mgr.? 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T10:38:55.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:38:54 smithi175 ceph-mon[25806]: pgmap v304: 97 pgs: 97 active+clean; 303 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 6.6 MiB/s wr, 670 op/s 2024-09-21T10:38:55.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:38:54 smithi175 ceph-mon[25806]: Standby manager daemon smithi175.txvwrq restarted 2024-09-21T10:38:55.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:38:54 smithi175 ceph-mon[25806]: Standby manager daemon smithi175.txvwrq started 2024-09-21T10:38:55.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:38:54 smithi175 ceph-mon[25806]: from='mgr.? 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi175.txvwrq/crt"}]: dispatch 2024-09-21T10:38:55.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:38:54 smithi175 ceph-mon[25806]: from='mgr.? 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T10:38:55.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:38:54 smithi175 ceph-mon[25806]: from='mgr.? 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi175.txvwrq/key"}]: dispatch 2024-09-21T10:38:55.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:38:54 smithi175 ceph-mon[25806]: from='mgr.? 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T10:38:55.800 INFO:teuthology.orchestra.run.smithi123.stdout: 5 58024 14.24 MB/sec warmup 116 sec latency 50.955 ms 2024-09-21T10:38:55.857 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:38:55 smithi123 ceph-mon[23709]: mgrmap e22: smithi123.stnuit(active, since 8m), standbys: smithi175.txvwrq 2024-09-21T10:38:56.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:38:55 smithi175 ceph-mon[25806]: mgrmap e22: smithi123.stnuit(active, since 8m), standbys: smithi175.txvwrq 2024-09-21T10:38:56.800 INFO:teuthology.orchestra.run.smithi123.stdout: 5 58912 14.24 MB/sec warmup 117 sec latency 23.967 ms 2024-09-21T10:38:56.863 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:38:56 smithi123 ceph-mon[23709]: pgmap v305: 97 pgs: 97 active+clean; 315 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 8.9 KiB/s rd, 8.7 MiB/s wr, 882 op/s 2024-09-21T10:38:57.125 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:38:56 smithi175 ceph-mon[25806]: pgmap v305: 97 pgs: 97 active+clean; 315 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 8.9 KiB/s rd, 8.7 MiB/s wr, 882 op/s 2024-09-21T10:38:57.800 INFO:teuthology.orchestra.run.smithi123.stdout: 5 59537 14.29 MB/sec warmup 118 sec latency 51.736 ms 2024-09-21T10:38:58.800 INFO:teuthology.orchestra.run.smithi123.stdout: 5 59896 14.27 MB/sec warmup 119 sec latency 152.033 ms 2024-09-21T10:38:58.960 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:38:58 smithi123 ceph-mon[23709]: pgmap v306: 97 pgs: 97 active+clean; 314 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 8.7 KiB/s rd, 7.6 MiB/s wr, 677 op/s 2024-09-21T10:38:59.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:38:58 smithi175 ceph-mon[25806]: pgmap v306: 97 pgs: 97 active+clean; 314 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 8.7 KiB/s rd, 7.6 MiB/s wr, 677 op/s 2024-09-21T10:39:00.799 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:00 smithi123 ceph-mon[23709]: pgmap v307: 97 pgs: 97 active+clean; 312 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 7.0 MiB/s wr, 661 op/s 2024-09-21T10:39:00.799 INFO:teuthology.orchestra.run.smithi123.stdout: 5 60546 9.59 MB/sec execute 1 sec latency 112.362 ms 2024-09-21T10:39:01.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:00 smithi175 ceph-mon[25806]: pgmap v307: 97 pgs: 97 active+clean; 312 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 7.0 MiB/s wr, 661 op/s 2024-09-21T10:39:01.800 INFO:teuthology.orchestra.run.smithi123.stdout: 5 60905 7.66 MB/sec execute 2 sec latency 44.264 ms 2024-09-21T10:39:02.800 INFO:teuthology.orchestra.run.smithi123.stdout: 5 61180 6.36 MB/sec execute 3 sec latency 83.593 ms 2024-09-21T10:39:02.929 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:02 smithi123 ceph-mon[23709]: pgmap v308: 97 pgs: 97 active+clean; 320 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 8.4 MiB/s wr, 815 op/s 2024-09-21T10:39:02.929 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:02 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:39:02.929 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:02 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:39:02.929 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:02 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:39:02.930 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:02 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:39:02.930 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:02 smithi123 ceph-mon[23709]: Upgrade: Need to upgrade myself (mgr.smithi123.stnuit) 2024-09-21T10:39:02.930 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:02 smithi123 ceph-mon[23709]: Failing over to other MGR 2024-09-21T10:39:02.930 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:02 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mgr fail", "who": "smithi123.stnuit"}]: dispatch 2024-09-21T10:39:02.930 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:02 smithi123 ceph-mon[23709]: osdmap e74: 8 total, 8 up, 8 in 2024-09-21T10:39:03.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:02 smithi175 ceph-mon[25806]: pgmap v308: 97 pgs: 97 active+clean; 320 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 8.4 MiB/s wr, 815 op/s 2024-09-21T10:39:03.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:02 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:39:03.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:02 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:39:03.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:02 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' 2024-09-21T10:39:03.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:02 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:39:03.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:02 smithi175 ceph-mon[25806]: Upgrade: Need to upgrade myself (mgr.smithi123.stnuit) 2024-09-21T10:39:03.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:02 smithi175 ceph-mon[25806]: Failing over to other MGR 2024-09-21T10:39:03.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:02 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mgr fail", "who": "smithi123.stnuit"}]: dispatch 2024-09-21T10:39:03.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:02 smithi175 ceph-mon[25806]: osdmap e74: 8 total, 8 up, 8 in 2024-09-21T10:39:03.800 INFO:teuthology.orchestra.run.smithi123.stdout: 5 61574 7.41 MB/sec execute 4 sec latency 44.356 ms 2024-09-21T10:39:03.945 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:03 smithi123 ceph-mon[23709]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "mgr fail", "who": "smithi123.stnuit"}]': finished 2024-09-21T10:39:03.945 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:03 smithi123 ceph-mon[23709]: mgrmap e23: smithi175.txvwrq(active, starting, since 0.994509s) 2024-09-21T10:39:03.945 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:03 smithi123 ceph-mon[23709]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "mon metadata", "id": "smithi123"}]: dispatch 2024-09-21T10:39:03.945 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:03 smithi123 ceph-mon[23709]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "mon metadata", "id": "smithi175"}]: dispatch 2024-09-21T10:39:03.945 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:03 smithi123 ceph-mon[23709]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi123.uqoyow"}]: dispatch 2024-09-21T10:39:03.945 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:03 smithi123 ceph-mon[23709]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi175.jrlevw"}]: dispatch 2024-09-21T10:39:03.945 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:03 smithi123 ceph-mon[23709]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "mgr metadata", "who": "smithi175.txvwrq", "id": "smithi175.txvwrq"}]: dispatch 2024-09-21T10:39:03.946 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:03 smithi123 ceph-mon[23709]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T10:39:03.946 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:03 smithi123 ceph-mon[23709]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T10:39:03.946 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:03 smithi123 ceph-mon[23709]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T10:39:03.946 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:03 smithi123 ceph-mon[23709]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T10:39:03.946 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:03 smithi123 ceph-mon[23709]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T10:39:03.946 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:03 smithi123 ceph-mon[23709]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T10:39:03.946 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:03 smithi123 ceph-mon[23709]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T10:39:03.946 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:03 smithi123 ceph-mon[23709]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T10:39:03.946 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:03 smithi123 ceph-mon[23709]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T10:39:03.946 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:03 smithi123 ceph-mon[23709]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T10:39:03.946 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:03 smithi123 ceph-mon[23709]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T10:39:03.946 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:03 smithi123 ceph-mon[23709]: Manager daemon smithi175.txvwrq is now available 2024-09-21T10:39:04.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:03 smithi175 ceph-mon[25806]: from='mgr.14202 172.21.15.123:0/2061986405' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "mgr fail", "who": "smithi123.stnuit"}]': finished 2024-09-21T10:39:04.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:03 smithi175 ceph-mon[25806]: mgrmap e23: smithi175.txvwrq(active, starting, since 0.994509s) 2024-09-21T10:39:04.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:03 smithi175 ceph-mon[25806]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "mon metadata", "id": "smithi123"}]: dispatch 2024-09-21T10:39:04.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:03 smithi175 ceph-mon[25806]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "mon metadata", "id": "smithi175"}]: dispatch 2024-09-21T10:39:04.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:03 smithi175 ceph-mon[25806]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi123.uqoyow"}]: dispatch 2024-09-21T10:39:04.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:03 smithi175 ceph-mon[25806]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi175.jrlevw"}]: dispatch 2024-09-21T10:39:04.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:03 smithi175 ceph-mon[25806]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "mgr metadata", "who": "smithi175.txvwrq", "id": "smithi175.txvwrq"}]: dispatch 2024-09-21T10:39:04.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:03 smithi175 ceph-mon[25806]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T10:39:04.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:03 smithi175 ceph-mon[25806]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T10:39:04.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:03 smithi175 ceph-mon[25806]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T10:39:04.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:03 smithi175 ceph-mon[25806]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T10:39:04.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:03 smithi175 ceph-mon[25806]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T10:39:04.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:03 smithi175 ceph-mon[25806]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T10:39:04.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:03 smithi175 ceph-mon[25806]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T10:39:04.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:03 smithi175 ceph-mon[25806]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T10:39:04.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:03 smithi175 ceph-mon[25806]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T10:39:04.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:03 smithi175 ceph-mon[25806]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T10:39:04.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:03 smithi175 ceph-mon[25806]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T10:39:04.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:03 smithi175 ceph-mon[25806]: Manager daemon smithi175.txvwrq is now available 2024-09-21T10:39:04.800 INFO:teuthology.orchestra.run.smithi123.stdout: 5 62151 7.81 MB/sec execute 5 sec latency 57.740 ms 2024-09-21T10:39:04.953 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:04 smithi123 ceph-mon[23709]: mgrmap e24: smithi175.txvwrq(active, since 2s) 2024-09-21T10:39:04.953 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:04 smithi123 ceph-mon[23709]: pgmap v3: 97 pgs: 97 active+clean; 330 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:39:04.953 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:04 smithi123 ceph-mon[23709]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:39:04.953 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:04 smithi123 ceph-mon[23709]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:39:04.953 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:04 smithi123 ceph-mon[23709]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:39:04.953 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:04 smithi123 ceph-mon[23709]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:39:04.953 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:04 smithi123 ceph-mon[23709]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:39:04.953 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:04 smithi123 ceph-mon[23709]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi175.txvwrq/mirror_snapshot_schedule"}]: dispatch 2024-09-21T10:39:04.953 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:04 smithi123 ceph-mon[23709]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T10:39:04.953 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:04 smithi123 ceph-mon[23709]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi175.txvwrq/mirror_snapshot_schedule"}]: dispatch 2024-09-21T10:39:04.953 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:04 smithi123 ceph-mon[23709]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T10:39:04.954 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:04 smithi123 ceph-mon[23709]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi175.txvwrq/trash_purge_schedule"}]: dispatch 2024-09-21T10:39:04.954 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:04 smithi123 ceph-mon[23709]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi175.txvwrq/trash_purge_schedule"}]: dispatch 2024-09-21T10:39:05.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:04 smithi175 ceph-mon[25806]: mgrmap e24: smithi175.txvwrq(active, since 2s) 2024-09-21T10:39:05.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:04 smithi175 ceph-mon[25806]: pgmap v3: 97 pgs: 97 active+clean; 330 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:39:05.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:04 smithi175 ceph-mon[25806]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:39:05.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:04 smithi175 ceph-mon[25806]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:39:05.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:04 smithi175 ceph-mon[25806]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:39:05.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:04 smithi175 ceph-mon[25806]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:39:05.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:04 smithi175 ceph-mon[25806]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:39:05.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:04 smithi175 ceph-mon[25806]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi175.txvwrq/mirror_snapshot_schedule"}]: dispatch 2024-09-21T10:39:05.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:04 smithi175 ceph-mon[25806]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T10:39:05.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:04 smithi175 ceph-mon[25806]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi175.txvwrq/mirror_snapshot_schedule"}]: dispatch 2024-09-21T10:39:05.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:04 smithi175 ceph-mon[25806]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T10:39:05.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:04 smithi175 ceph-mon[25806]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi175.txvwrq/trash_purge_schedule"}]: dispatch 2024-09-21T10:39:05.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:04 smithi175 ceph-mon[25806]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi175.txvwrq/trash_purge_schedule"}]: dispatch 2024-09-21T10:39:05.800 INFO:teuthology.orchestra.run.smithi123.stdout: 5 62529 7.87 MB/sec execute 6 sec latency 196.713 ms 2024-09-21T10:39:05.967 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:05 smithi123 ceph-mon[23709]: Checking for cert/key for grafana.smithi123 2024-09-21T10:39:05.967 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:05 smithi123 ceph-mon[23709]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T10:39:05.967 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:05 smithi123 ceph-mon[23709]: pgmap v4: 97 pgs: 97 active+clean; 330 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:39:05.967 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:05 smithi123 ceph-mon[23709]: Deploying cephadm binary to smithi175 2024-09-21T10:39:06.099 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:05 smithi175 ceph-mon[25806]: Checking for cert/key for grafana.smithi123 2024-09-21T10:39:06.099 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:05 smithi175 ceph-mon[25806]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T10:39:06.100 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:05 smithi175 ceph-mon[25806]: pgmap v4: 97 pgs: 97 active+clean; 330 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:39:06.100 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:05 smithi175 ceph-mon[25806]: Deploying cephadm binary to smithi175 2024-09-21T10:39:06.800 INFO:teuthology.orchestra.run.smithi123.stdout: 5 63094 9.38 MB/sec execute 7 sec latency 39.954 ms 2024-09-21T10:39:06.975 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:06 smithi123 ceph-mon[23709]: Deploying cephadm binary to smithi123 2024-09-21T10:39:06.975 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:06 smithi123 ceph-mon[23709]: [21/Sep/2024:10:39:05] ENGINE Bus STARTING 2024-09-21T10:39:06.975 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:06 smithi123 ceph-mon[23709]: [21/Sep/2024:10:39:06] ENGINE Serving on https://172.21.15.175:7150 2024-09-21T10:39:06.975 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:06 smithi123 ceph-mon[23709]: [21/Sep/2024:10:39:06] ENGINE Error in HTTPServer.serve 2024-09-21T10:39:06.975 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Traceback (most recent call last): 2024-09-21T10:39:06.975 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T10:39:06.976 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: self._connections.run(self.expiration_interval) 2024-09-21T10:39:06.976 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T10:39:06.976 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: self._run(expiration_interval) 2024-09-21T10:39:06.976 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T10:39:06.976 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T10:39:06.976 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T10:39:06.976 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T10:39:06.976 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T10:39:06.976 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: s = self.context.wrap_socket( 2024-09-21T10:39:06.976 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T10:39:06.976 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: return self.sslsocket_class._create( 2024-09-21T10:39:06.976 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T10:39:06.976 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: self.do_handshake() 2024-09-21T10:39:06.976 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T10:39:06.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: self._sslobj.do_handshake() 2024-09-21T10:39:06.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T10:39:07.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:06 smithi175 ceph-mon[25806]: Deploying cephadm binary to smithi123 2024-09-21T10:39:07.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:06 smithi175 ceph-mon[25806]: [21/Sep/2024:10:39:05] ENGINE Bus STARTING 2024-09-21T10:39:07.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:06 smithi175 ceph-mon[25806]: [21/Sep/2024:10:39:06] ENGINE Serving on https://172.21.15.175:7150 2024-09-21T10:39:07.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:06 smithi175 ceph-mon[25806]: [21/Sep/2024:10:39:06] ENGINE Error in HTTPServer.serve 2024-09-21T10:39:07.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: Traceback (most recent call last): 2024-09-21T10:39:07.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T10:39:07.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: self._connections.run(self.expiration_interval) 2024-09-21T10:39:07.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T10:39:07.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: self._run(expiration_interval) 2024-09-21T10:39:07.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T10:39:07.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T10:39:07.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T10:39:07.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T10:39:07.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T10:39:07.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: s = self.context.wrap_socket( 2024-09-21T10:39:07.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T10:39:07.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: return self.sslsocket_class._create( 2024-09-21T10:39:07.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T10:39:07.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: self.do_handshake() 2024-09-21T10:39:07.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T10:39:07.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: self._sslobj.do_handshake() 2024-09-21T10:39:07.125 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T10:39:07.456 INFO:teuthology.orchestra.run.smithi123.stdout:true 2024-09-21T10:39:07.801 INFO:teuthology.orchestra.run.smithi123.stdout: 5 63459 10.15 MB/sec execute 8 sec latency 50.841 ms 2024-09-21T10:39:07.986 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:07 smithi123 ceph-mon[23709]: [21/Sep/2024:10:39:06] ENGINE Serving on http://172.21.15.175:8765 2024-09-21T10:39:07.986 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:07 smithi123 ceph-mon[23709]: [21/Sep/2024:10:39:06] ENGINE Bus STARTED 2024-09-21T10:39:07.987 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:07 smithi123 ceph-mon[23709]: pgmap v5: 97 pgs: 97 active+clean; 330 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:39:07.987 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:07 smithi123 ceph-mon[23709]: mgrmap e25: smithi175.txvwrq(active, since 5s) 2024-09-21T10:39:08.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:07 smithi175 ceph-mon[25806]: [21/Sep/2024:10:39:06] ENGINE Serving on http://172.21.15.175:8765 2024-09-21T10:39:08.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:07 smithi175 ceph-mon[25806]: [21/Sep/2024:10:39:06] ENGINE Bus STARTED 2024-09-21T10:39:08.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:07 smithi175 ceph-mon[25806]: pgmap v5: 97 pgs: 97 active+clean; 330 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:39:08.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:07 smithi175 ceph-mon[25806]: mgrmap e25: smithi175.txvwrq(active, since 5s) 2024-09-21T10:39:08.699 INFO:teuthology.orchestra.run.smithi123.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T10:39:08.699 INFO:teuthology.orchestra.run.smithi123.stdout:alertmanager.smithi123 smithi123 *:9093,9094 running (7m) 2m ago 8m 22.5M - ba2b418f427c db0d7d5d885a 2024-09-21T10:39:08.699 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi123 smithi123 running (8m) 2m ago 8m 6405k - 17.2.0 e1d6a67b021e c459f87d5b50 2024-09-21T10:39:08.699 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi175 smithi175 running (8m) 17s ago 8m 6405k - 17.2.0 e1d6a67b021e e35e422d5583 2024-09-21T10:39:08.699 INFO:teuthology.orchestra.run.smithi123.stdout:grafana.smithi123 smithi123 *:3000 running (7m) 2m ago 8m 53.9M - 8.3.5 dad864ee21e9 86ee7306dede 2024-09-21T10:39:08.699 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi123.uqoyow smithi123 running (2m) 2m ago 2m 15.9M - 17.2.0 e1d6a67b021e f0a21eaac266 2024-09-21T10:39:08.700 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi175.jrlevw smithi175 running (2m) 17s ago 2m 11.6M - 17.2.0 e1d6a67b021e b21b16f4bc50 2024-09-21T10:39:08.700 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi123.stnuit smithi123 *:9283 running (10m) 2m ago 10m 459M - 17.2.0 e1d6a67b021e 417fa41b561b 2024-09-21T10:39:08.700 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi175.txvwrq smithi175 *:8443,9283 running (22s) 17s ago 8m 64.4M - 19.3.0-5103-g5c1cdc3c 8f4d013db793 b0be7f732c0b 2024-09-21T10:39:08.700 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi123 smithi123 running (10m) 2m ago 10m 46.6M 2048M 17.2.0 e1d6a67b021e 6ad9a8bc53b7 2024-09-21T10:39:08.700 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi175 smithi175 running (8m) 17s ago 8m 40.3M 2048M 17.2.0 e1d6a67b021e 753f564c8cf5 2024-09-21T10:39:08.700 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.0.0.smithi123.ddatpr smithi123 *:2049 running (2m) 2m ago 2m 49.1M - 4.0 e1d6a67b021e 36f8cb2281eb 2024-09-21T10:39:08.700 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.1.0.smithi175.wpawer smithi175 *:2049 running (2m) 17s ago 2m 77.5M - 4.0 e1d6a67b021e 2200aeed9200 2024-09-21T10:39:08.700 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi123 smithi123 *:9100 running (8m) 2m ago 8m 19.7M - 1dbe0e931976 875d5b2d6936 2024-09-21T10:39:08.700 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi175 smithi175 *:9100 running (7m) 17s ago 7m 19.6M - 1dbe0e931976 92ddc27e1ca2 2024-09-21T10:39:08.700 INFO:teuthology.orchestra.run.smithi123.stdout:osd.0 smithi123 running (7m) 2m ago 7m 43.0M 3216M 17.2.0 e1d6a67b021e a94edf88c456 2024-09-21T10:39:08.700 INFO:teuthology.orchestra.run.smithi123.stdout:osd.1 smithi123 running (6m) 2m ago 6m 45.5M 3216M 17.2.0 e1d6a67b021e 59aee515b0f8 2024-09-21T10:39:08.700 INFO:teuthology.orchestra.run.smithi123.stdout:osd.2 smithi123 running (6m) 2m ago 6m 40.5M 3216M 17.2.0 e1d6a67b021e 7db93f8ce5b5 2024-09-21T10:39:08.700 INFO:teuthology.orchestra.run.smithi123.stdout:osd.3 smithi123 running (5m) 2m ago 5m 39.8M 3216M 17.2.0 e1d6a67b021e c1672dea7b1f 2024-09-21T10:39:08.701 INFO:teuthology.orchestra.run.smithi123.stdout:osd.4 smithi175 running (5m) 17s ago 5m 221M 3984M 17.2.0 e1d6a67b021e 48fe728c259e 2024-09-21T10:39:08.701 INFO:teuthology.orchestra.run.smithi123.stdout:osd.5 smithi175 running (5m) 17s ago 4m 231M 3984M 17.2.0 e1d6a67b021e e270d5a21b14 2024-09-21T10:39:08.701 INFO:teuthology.orchestra.run.smithi123.stdout:osd.6 smithi175 running (4m) 17s ago 4m 211M 3984M 17.2.0 e1d6a67b021e 9bcbf018eb20 2024-09-21T10:39:08.701 INFO:teuthology.orchestra.run.smithi123.stdout:osd.7 smithi175 running (4m) 17s ago 4m 316M 3984M 17.2.0 e1d6a67b021e e9b8f1f90994 2024-09-21T10:39:08.701 INFO:teuthology.orchestra.run.smithi123.stdout:prometheus.smithi123 smithi123 *:9095 running (7m) 2m ago 7m 59.9M - 514e6a882f6e 2beddd4ece2e 2024-09-21T10:39:08.800 INFO:teuthology.orchestra.run.smithi123.stdout: 5 63906 11.19 MB/sec execute 9 sec latency 35.601 ms 2024-09-21T10:39:08.989 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:08 smithi123 ceph-mon[23709]: from='client.14706 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:39:08.989 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:08 smithi123 ceph-mon[23709]: from='client.24447 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:39:09.095 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:08 smithi175 ceph-mon[25806]: from='client.14706 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:39:09.095 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:08 smithi175 ceph-mon[25806]: from='client.24447 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:39:09.469 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-09-21T10:39:09.469 INFO:teuthology.orchestra.run.smithi123.stdout: "mon": { 2024-09-21T10:39:09.469 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T10:39:09.470 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:39:09.470 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr": { 2024-09-21T10:39:09.470 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev)": 1 2024-09-21T10:39:09.470 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:39:09.470 INFO:teuthology.orchestra.run.smithi123.stdout: "osd": { 2024-09-21T10:39:09.470 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-21T10:39:09.470 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:39:09.470 INFO:teuthology.orchestra.run.smithi123.stdout: "mds": { 2024-09-21T10:39:09.470 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T10:39:09.470 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:39:09.470 INFO:teuthology.orchestra.run.smithi123.stdout: "overall": { 2024-09-21T10:39:09.470 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-09-21T10:39:09.471 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev)": 1 2024-09-21T10:39:09.471 INFO:teuthology.orchestra.run.smithi123.stdout: } 2024-09-21T10:39:09.471 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-09-21T10:39:09.801 INFO:teuthology.orchestra.run.smithi123.stdout: 5 64246 10.86 MB/sec execute 10 sec latency 44.325 ms 2024-09-21T10:39:10.011 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:09 smithi123 ceph-mon[23709]: from='client.24451 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:39:10.011 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:09 smithi123 ceph-mon[23709]: pgmap v6: 97 pgs: 97 active+clean; 333 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 8.4 KiB/s rd, 2.4 MiB/s wr, 376 op/s 2024-09-21T10:39:10.011 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:09 smithi123 ceph-mon[23709]: mgrmap e26: smithi175.txvwrq(active, since 7s) 2024-09-21T10:39:10.011 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:09 smithi123 ceph-mon[23709]: from='client.? 172.21.15.123:0/3460368686' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T10:39:10.086 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-09-21T10:39:10.086 INFO:teuthology.orchestra.run.smithi123.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472", 2024-09-21T10:39:10.086 INFO:teuthology.orchestra.run.smithi123.stdout: "in_progress": true, 2024-09-21T10:39:10.086 INFO:teuthology.orchestra.run.smithi123.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T10:39:10.086 INFO:teuthology.orchestra.run.smithi123.stdout: "services_complete": [], 2024-09-21T10:39:10.086 INFO:teuthology.orchestra.run.smithi123.stdout: "progress": "1/23 daemons upgraded", 2024-09-21T10:39:10.086 INFO:teuthology.orchestra.run.smithi123.stdout: "message": "", 2024-09-21T10:39:10.086 INFO:teuthology.orchestra.run.smithi123.stdout: "is_paused": false 2024-09-21T10:39:10.086 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-09-21T10:39:10.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:09 smithi175 ceph-mon[25806]: from='client.24451 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:39:10.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:09 smithi175 ceph-mon[25806]: pgmap v6: 97 pgs: 97 active+clean; 333 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 8.4 KiB/s rd, 2.4 MiB/s wr, 376 op/s 2024-09-21T10:39:10.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:09 smithi175 ceph-mon[25806]: mgrmap e26: smithi175.txvwrq(active, since 7s) 2024-09-21T10:39:10.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:09 smithi175 ceph-mon[25806]: from='client.? 172.21.15.123:0/3460368686' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T10:39:10.712 INFO:teuthology.orchestra.run.smithi123.stdout:HEALTH_OK 2024-09-21T10:39:10.801 INFO:teuthology.orchestra.run.smithi123.stdout: 5 64530 10.13 MB/sec execute 11 sec latency 110.727 ms 2024-09-21T10:39:10.998 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:10 smithi175 ceph-mon[25806]: from='client.24459 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:39:10.999 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:10 smithi175 ceph-mon[25806]: from='client.? 172.21.15.123:0/4292586609' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T10:39:11.017 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:10 smithi123 ceph-mon[23709]: from='client.24459 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:39:11.017 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:10 smithi123 ceph-mon[23709]: from='client.? 172.21.15.123:0/4292586609' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T10:39:11.800 INFO:teuthology.orchestra.run.smithi123.stdout: 5 64851 9.76 MB/sec execute 12 sec latency 49.888 ms 2024-09-21T10:39:11.941 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:11 smithi123 ceph-mon[23709]: pgmap v7: 97 pgs: 97 active+clean; 336 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 31 KiB/s rd, 4.8 MiB/s wr, 502 op/s 2024-09-21T10:39:11.941 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:11 smithi123 ceph-mon[23709]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:39:11.941 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:11 smithi123 ceph-mon[23709]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:39:11.941 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:11 smithi123 ceph-mon[23709]: Standby manager daemon smithi123.stnuit started 2024-09-21T10:39:11.941 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:11 smithi123 ceph-mon[23709]: from='mgr.? 172.21.15.123:0/1319182742' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi123.stnuit/crt"}]: dispatch 2024-09-21T10:39:11.941 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:11 smithi123 ceph-mon[23709]: from='mgr.? 172.21.15.123:0/1319182742' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T10:39:11.941 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:11 smithi123 ceph-mon[23709]: from='mgr.? 172.21.15.123:0/1319182742' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi123.stnuit/key"}]: dispatch 2024-09-21T10:39:11.942 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:11 smithi123 ceph-mon[23709]: from='mgr.? 172.21.15.123:0/1319182742' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T10:39:12.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:11 smithi175 ceph-mon[25806]: pgmap v7: 97 pgs: 97 active+clean; 336 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 31 KiB/s rd, 4.8 MiB/s wr, 502 op/s 2024-09-21T10:39:12.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:11 smithi175 ceph-mon[25806]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:39:12.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:11 smithi175 ceph-mon[25806]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:39:12.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:11 smithi175 ceph-mon[25806]: Standby manager daemon smithi123.stnuit started 2024-09-21T10:39:12.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:11 smithi175 ceph-mon[25806]: from='mgr.? 172.21.15.123:0/1319182742' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi123.stnuit/crt"}]: dispatch 2024-09-21T10:39:12.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:11 smithi175 ceph-mon[25806]: from='mgr.? 172.21.15.123:0/1319182742' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T10:39:12.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:11 smithi175 ceph-mon[25806]: from='mgr.? 172.21.15.123:0/1319182742' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi123.stnuit/key"}]: dispatch 2024-09-21T10:39:12.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:11 smithi175 ceph-mon[25806]: from='mgr.? 172.21.15.123:0/1319182742' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T10:39:12.800 INFO:teuthology.orchestra.run.smithi123.stdout: 5 65251 9.89 MB/sec execute 13 sec latency 59.154 ms 2024-09-21T10:39:13.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:13 smithi175 ceph-mon[25806]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:39:13.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:13 smithi175 ceph-mon[25806]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:39:13.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:13 smithi175 ceph-mon[25806]: mgrmap e27: smithi175.txvwrq(active, since 10s), standbys: smithi123.stnuit 2024-09-21T10:39:13.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:13 smithi175 ceph-mon[25806]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "mgr metadata", "who": "smithi123.stnuit", "id": "smithi123.stnuit"}]: dispatch 2024-09-21T10:39:13.318 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:13 smithi123 ceph-mon[23709]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:39:13.318 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:13 smithi123 ceph-mon[23709]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:39:13.318 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:13 smithi123 ceph-mon[23709]: mgrmap e27: smithi175.txvwrq(active, since 10s), standbys: smithi123.stnuit 2024-09-21T10:39:13.318 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:13 smithi123 ceph-mon[23709]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "mgr metadata", "who": "smithi123.stnuit", "id": "smithi123.stnuit"}]: dispatch 2024-09-21T10:39:13.800 INFO:teuthology.orchestra.run.smithi123.stdout: 5 65713 9.64 MB/sec execute 14 sec latency 67.417 ms 2024-09-21T10:39:14.337 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:14 smithi123 ceph-mon[23709]: pgmap v8: 97 pgs: 97 active+clean; 336 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 3.8 MiB/s wr, 390 op/s 2024-09-21T10:39:14.354 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:14 smithi175 ceph-mon[25806]: pgmap v8: 97 pgs: 97 active+clean; 336 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 3.8 MiB/s wr, 390 op/s 2024-09-21T10:39:14.800 INFO:teuthology.orchestra.run.smithi123.stdout: 5 66146 9.63 MB/sec execute 15 sec latency 46.038 ms 2024-09-21T10:39:15.423 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:15 smithi123 ceph-mon[23709]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:39:15.424 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:15 smithi123 ceph-mon[23709]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:39:15.424 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:15 smithi123 ceph-mon[23709]: pgmap v9: 97 pgs: 97 active+clean; 335 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 5.2 MiB/s wr, 619 op/s 2024-09-21T10:39:15.424 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:15 smithi123 ceph-mon[23709]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:39:15.424 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:15 smithi123 ceph-mon[23709]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:39:15.424 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:15 smithi123 ceph-mon[23709]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T10:39:15.424 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:15 smithi123 ceph-mon[23709]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T10:39:15.424 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:15 smithi123 ceph-mon[23709]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T10:39:15.424 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:15 smithi123 ceph-mon[23709]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T10:39:15.424 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:15 smithi123 ceph-mon[23709]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T10:39:15.424 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:15 smithi123 ceph-mon[23709]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T10:39:15.424 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:15 smithi123 ceph-mon[23709]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T10:39:15.425 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:15 smithi123 ceph-mon[23709]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T10:39:15.425 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:15 smithi123 ceph-mon[23709]: Adjusting osd_memory_target on smithi123 to 1936M 2024-09-21T10:39:15.425 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:15 smithi123 ceph-mon[23709]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:39:15.621 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:15 smithi175 ceph-mon[25806]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:39:15.621 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:15 smithi175 ceph-mon[25806]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:39:15.621 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:15 smithi175 ceph-mon[25806]: pgmap v9: 97 pgs: 97 active+clean; 335 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 5.2 MiB/s wr, 619 op/s 2024-09-21T10:39:15.621 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:15 smithi175 ceph-mon[25806]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:39:15.622 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:15 smithi175 ceph-mon[25806]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:39:15.622 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:15 smithi175 ceph-mon[25806]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T10:39:15.622 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:15 smithi175 ceph-mon[25806]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T10:39:15.622 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:15 smithi175 ceph-mon[25806]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T10:39:15.622 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:15 smithi175 ceph-mon[25806]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T10:39:15.622 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:15 smithi175 ceph-mon[25806]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T10:39:15.622 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:15 smithi175 ceph-mon[25806]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T10:39:15.622 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:15 smithi175 ceph-mon[25806]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T10:39:15.622 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:15 smithi175 ceph-mon[25806]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T10:39:15.622 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:15 smithi175 ceph-mon[25806]: Adjusting osd_memory_target on smithi123 to 1936M 2024-09-21T10:39:15.622 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:15 smithi175 ceph-mon[25806]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:39:15.800 INFO:teuthology.orchestra.run.smithi123.stdout: 5 66761 10.44 MB/sec execute 16 sec latency 31.117 ms 2024-09-21T10:39:16.801 INFO:teuthology.orchestra.run.smithi123.stdout: 5 67478 11.59 MB/sec execute 17 sec latency 16.940 ms 2024-09-21T10:39:17.182 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:17 smithi123 ceph-mon[23709]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:39:17.182 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:17 smithi123 ceph-mon[23709]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:39:17.182 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:17 smithi123 ceph-mon[23709]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T10:39:17.182 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:17 smithi123 ceph-mon[23709]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T10:39:17.182 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:17 smithi123 ceph-mon[23709]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T10:39:17.182 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:17 smithi123 ceph-mon[23709]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T10:39:17.182 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:17 smithi123 ceph-mon[23709]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T10:39:17.182 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:17 smithi123 ceph-mon[23709]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T10:39:17.182 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:17 smithi123 ceph-mon[23709]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T10:39:17.182 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:17 smithi123 ceph-mon[23709]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T10:39:17.182 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:17 smithi123 ceph-mon[23709]: Adjusting osd_memory_target on smithi175 to 2704M 2024-09-21T10:39:17.182 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:17 smithi123 ceph-mon[23709]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:39:17.183 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:17 smithi123 ceph-mon[23709]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:39:17.183 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:17 smithi123 ceph-mon[23709]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T10:39:17.183 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:17 smithi123 ceph-mon[23709]: Updating smithi123:/etc/ceph/ceph.conf 2024-09-21T10:39:17.183 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:17 smithi123 ceph-mon[23709]: Updating smithi175:/etc/ceph/ceph.conf 2024-09-21T10:39:17.183 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:17 smithi123 ceph-mon[23709]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:39:17.183 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:17 smithi123 ceph-mon[23709]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:39:17.183 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:17 smithi123 ceph-mon[23709]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:39:17.183 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:17 smithi123 ceph-mon[23709]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:39:17.183 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:17 smithi123 ceph-mon[23709]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:39:17.183 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:17 smithi123 ceph-mon[23709]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:39:17.183 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:17 smithi123 ceph-mon[23709]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:39:17.183 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:17 smithi123 ceph-mon[23709]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:39:17.183 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:17 smithi123 ceph-mon[23709]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:39:17.184 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:17 smithi123 ceph-mon[23709]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:39:17.626 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:17 smithi175 ceph-mon[25806]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:39:17.630 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:17 smithi175 ceph-mon[25806]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:39:17.630 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:17 smithi175 ceph-mon[25806]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T10:39:17.631 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:17 smithi175 ceph-mon[25806]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T10:39:17.631 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:17 smithi175 ceph-mon[25806]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T10:39:17.631 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:17 smithi175 ceph-mon[25806]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T10:39:17.631 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:17 smithi175 ceph-mon[25806]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T10:39:17.631 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:17 smithi175 ceph-mon[25806]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T10:39:17.631 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:17 smithi175 ceph-mon[25806]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T10:39:17.631 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:17 smithi175 ceph-mon[25806]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T10:39:17.631 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:17 smithi175 ceph-mon[25806]: Adjusting osd_memory_target on smithi175 to 2704M 2024-09-21T10:39:17.631 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:17 smithi175 ceph-mon[25806]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:39:17.632 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:17 smithi175 ceph-mon[25806]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:39:17.632 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:17 smithi175 ceph-mon[25806]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T10:39:17.632 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:17 smithi175 ceph-mon[25806]: Updating smithi123:/etc/ceph/ceph.conf 2024-09-21T10:39:17.632 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:17 smithi175 ceph-mon[25806]: Updating smithi175:/etc/ceph/ceph.conf 2024-09-21T10:39:17.632 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:17 smithi175 ceph-mon[25806]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:39:17.632 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:17 smithi175 ceph-mon[25806]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:39:17.632 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:17 smithi175 ceph-mon[25806]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:39:17.632 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:17 smithi175 ceph-mon[25806]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:39:17.632 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:17 smithi175 ceph-mon[25806]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:39:17.632 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:17 smithi175 ceph-mon[25806]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:39:17.632 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:17 smithi175 ceph-mon[25806]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:39:17.633 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:17 smithi175 ceph-mon[25806]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:39:17.633 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:17 smithi175 ceph-mon[25806]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:39:17.633 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:17 smithi175 ceph-mon[25806]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:39:17.801 INFO:teuthology.orchestra.run.smithi123.stdout: 5 67969 11.46 MB/sec execute 18 sec latency 32.306 ms 2024-09-21T10:39:18.191 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:18 smithi123 ceph-mon[23709]: Updating smithi175:/var/lib/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T10:39:18.191 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:18 smithi123 ceph-mon[23709]: Updating smithi123:/var/lib/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T10:39:18.191 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:18 smithi123 ceph-mon[23709]: Updating smithi175:/etc/ceph/ceph.client.admin.keyring 2024-09-21T10:39:18.192 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:18 smithi123 ceph-mon[23709]: Updating smithi123:/etc/ceph/ceph.client.admin.keyring 2024-09-21T10:39:18.192 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:18 smithi123 ceph-mon[23709]: Updating smithi175:/var/lib/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T10:39:18.192 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:18 smithi123 ceph-mon[23709]: Updating smithi123:/var/lib/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T10:39:18.192 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:18 smithi123 ceph-mon[23709]: pgmap v10: 97 pgs: 97 active+clean; 335 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 4.7 MiB/s wr, 567 op/s 2024-09-21T10:39:18.192 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:18 smithi123 ceph-mon[23709]: Reconfiguring alertmanager.smithi123 (dependencies changed)... 2024-09-21T10:39:18.192 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:18 smithi123 ceph-mon[23709]: Deploying daemon alertmanager.smithi123 on smithi123 2024-09-21T10:39:18.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:18 smithi175 ceph-mon[25806]: Updating smithi175:/var/lib/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T10:39:18.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:18 smithi175 ceph-mon[25806]: Updating smithi123:/var/lib/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T10:39:18.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:18 smithi175 ceph-mon[25806]: Updating smithi175:/etc/ceph/ceph.client.admin.keyring 2024-09-21T10:39:18.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:18 smithi175 ceph-mon[25806]: Updating smithi123:/etc/ceph/ceph.client.admin.keyring 2024-09-21T10:39:18.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:18 smithi175 ceph-mon[25806]: Updating smithi175:/var/lib/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T10:39:18.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:18 smithi175 ceph-mon[25806]: Updating smithi123:/var/lib/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T10:39:18.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:18 smithi175 ceph-mon[25806]: pgmap v10: 97 pgs: 97 active+clean; 335 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 4.7 MiB/s wr, 567 op/s 2024-09-21T10:39:18.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:18 smithi175 ceph-mon[25806]: Reconfiguring alertmanager.smithi123 (dependencies changed)... 2024-09-21T10:39:18.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:18 smithi175 ceph-mon[25806]: Deploying daemon alertmanager.smithi123 on smithi123 2024-09-21T10:39:18.801 INFO:teuthology.orchestra.run.smithi123.stdout: 5 68343 11.16 MB/sec execute 19 sec latency 40.701 ms 2024-09-21T10:39:19.801 INFO:teuthology.orchestra.run.smithi123.stdout: 5 68826 11.23 MB/sec execute 20 sec latency 37.213 ms 2024-09-21T10:39:20.225 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:20 smithi123 ceph-mon[23709]: pgmap v11: 97 pgs: 97 active+clean; 344 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 6.4 MiB/s wr, 810 op/s 2024-09-21T10:39:20.225 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:20 smithi123 ceph-mon[23709]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T10:39:20.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:20 smithi175 ceph-mon[25806]: pgmap v11: 97 pgs: 97 active+clean; 344 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 6.4 MiB/s wr, 810 op/s 2024-09-21T10:39:20.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:20 smithi175 ceph-mon[25806]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T10:39:20.801 INFO:teuthology.orchestra.run.smithi123.stdout: 5 69601 11.32 MB/sec execute 21 sec latency 55.040 ms 2024-09-21T10:39:21.735 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:21 smithi123 ceph-mon[23709]: pgmap v12: 97 pgs: 97 active+clean; 348 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 7.4 MiB/s wr, 726 op/s 2024-09-21T10:39:21.801 INFO:teuthology.orchestra.run.smithi123.stdout: 5 70115 11.63 MB/sec execute 22 sec latency 42.669 ms 2024-09-21T10:39:22.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:21 smithi175 ceph-mon[25806]: pgmap v12: 97 pgs: 97 active+clean; 348 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 7.4 MiB/s wr, 726 op/s 2024-09-21T10:39:22.801 INFO:teuthology.orchestra.run.smithi123.stdout: 5 70613 11.80 MB/sec execute 23 sec latency 95.012 ms 2024-09-21T10:39:23.801 INFO:teuthology.orchestra.run.smithi123.stdout: 5 71071 12.24 MB/sec execute 24 sec latency 35.384 ms 2024-09-21T10:39:23.848 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:23 smithi123 ceph-mon[23709]: pgmap v13: 97 pgs: 97 active+clean; 348 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.6 MiB/s wr, 590 op/s 2024-09-21T10:39:24.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:23 smithi175 ceph-mon[25806]: pgmap v13: 97 pgs: 97 active+clean; 348 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.6 MiB/s wr, 590 op/s 2024-09-21T10:39:24.801 INFO:teuthology.orchestra.run.smithi123.stdout: 5 71398 12.01 MB/sec execute 25 sec latency 47.369 ms 2024-09-21T10:39:25.801 INFO:teuthology.orchestra.run.smithi123.stdout: 5 71970 11.86 MB/sec execute 26 sec latency 70.581 ms 2024-09-21T10:39:25.862 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:25 smithi123 ceph-mon[23709]: pgmap v14: 97 pgs: 97 active+clean; 346 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.8 MiB/s wr, 903 op/s 2024-09-21T10:39:26.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:25 smithi175 ceph-mon[25806]: pgmap v14: 97 pgs: 97 active+clean; 346 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.8 MiB/s wr, 903 op/s 2024-09-21T10:39:26.801 INFO:teuthology.orchestra.run.smithi123.stdout: 5 72514 12.02 MB/sec execute 27 sec latency 37.534 ms 2024-09-21T10:39:27.801 INFO:teuthology.orchestra.run.smithi123.stdout: 5 73078 11.86 MB/sec execute 28 sec latency 41.491 ms 2024-09-21T10:39:27.899 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:27 smithi123 ceph-mon[23709]: pgmap v15: 97 pgs: 97 active+clean; 346 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.9 KiB/s rd, 6.9 MiB/s wr, 629 op/s 2024-09-21T10:39:28.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:27 smithi175 ceph-mon[25806]: pgmap v15: 97 pgs: 97 active+clean; 346 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.9 KiB/s rd, 6.9 MiB/s wr, 629 op/s 2024-09-21T10:39:28.801 INFO:teuthology.orchestra.run.smithi123.stdout: 5 73406 11.66 MB/sec execute 29 sec latency 73.563 ms 2024-09-21T10:39:29.801 INFO:teuthology.orchestra.run.smithi123.stdout: 5 73931 12.02 MB/sec execute 30 sec latency 33.153 ms 2024-09-21T10:39:29.915 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:29 smithi123 ceph-mon[23709]: pgmap v16: 97 pgs: 97 active+clean; 344 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 8.1 MiB/s wr, 822 op/s 2024-09-21T10:39:30.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:29 smithi175 ceph-mon[25806]: pgmap v16: 97 pgs: 97 active+clean; 344 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 8.1 MiB/s wr, 822 op/s 2024-09-21T10:39:30.802 INFO:teuthology.orchestra.run.smithi123.stdout: 5 74363 12.19 MB/sec execute 31 sec latency 42.020 ms 2024-09-21T10:39:31.706 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:31 smithi123 ceph-mon[23709]: pgmap v17: 97 pgs: 97 active+clean; 329 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 7.7 MiB/s wr, 671 op/s 2024-09-21T10:39:31.802 INFO:teuthology.orchestra.run.smithi123.stdout: 5 74690 12.23 MB/sec execute 32 sec latency 51.840 ms 2024-09-21T10:39:32.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:31 smithi175 ceph-mon[25806]: pgmap v17: 97 pgs: 97 active+clean; 329 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 7.7 MiB/s wr, 671 op/s 2024-09-21T10:39:32.803 INFO:teuthology.orchestra.run.smithi123.stdout: 5 74907 12.00 MB/sec execute 33 sec latency 90.810 ms 2024-09-21T10:39:33.802 INFO:teuthology.orchestra.run.smithi123.stdout: 5 75244 11.74 MB/sec execute 34 sec latency 68.108 ms 2024-09-21T10:39:34.012 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:33 smithi123 ceph-mon[23709]: pgmap v18: 97 pgs: 97 active+clean; 329 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 5.7 MiB/s wr, 598 op/s 2024-09-21T10:39:34.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:33 smithi175 ceph-mon[25806]: pgmap v18: 97 pgs: 97 active+clean; 329 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 5.7 MiB/s wr, 598 op/s 2024-09-21T10:39:34.802 INFO:teuthology.orchestra.run.smithi123.stdout: 5 75512 11.56 MB/sec execute 35 sec latency 50.690 ms 2024-09-21T10:39:35.802 INFO:teuthology.orchestra.run.smithi123.stdout: 5 75954 11.63 MB/sec execute 36 sec latency 38.248 ms 2024-09-21T10:39:35.806 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:35 smithi123 ceph-mon[23709]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:39:35.806 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:35 smithi123 ceph-mon[23709]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:39:35.806 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:35 smithi123 ceph-mon[23709]: Reconfiguring grafana.smithi123 (dependencies changed)... 2024-09-21T10:39:35.807 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:35 smithi123 ceph-mon[23709]: Regenerating cephadm self-signed grafana TLS certificates 2024-09-21T10:39:35.807 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:35 smithi123 ceph-mon[23709]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T10:39:35.807 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:35 smithi123 ceph-mon[23709]: pgmap v19: 97 pgs: 97 active+clean; 335 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.3 MiB/s wr, 807 op/s 2024-09-21T10:39:35.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:35 smithi175 ceph-mon[25806]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:39:35.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:35 smithi175 ceph-mon[25806]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:39:35.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:35 smithi175 ceph-mon[25806]: Reconfiguring grafana.smithi123 (dependencies changed)... 2024-09-21T10:39:35.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:35 smithi175 ceph-mon[25806]: Regenerating cephadm self-signed grafana TLS certificates 2024-09-21T10:39:35.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:35 smithi175 ceph-mon[25806]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T10:39:35.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:35 smithi175 ceph-mon[25806]: pgmap v19: 97 pgs: 97 active+clean; 335 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.3 MiB/s wr, 807 op/s 2024-09-21T10:39:36.802 INFO:teuthology.orchestra.run.smithi123.stdout: 5 76617 11.51 MB/sec execute 37 sec latency 25.403 ms 2024-09-21T10:39:36.821 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:36 smithi123 ceph-mon[23709]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:39:36.821 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:36 smithi123 ceph-mon[23709]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:39:36.821 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:36 smithi123 ceph-mon[23709]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-21T10:39:36.822 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:36 smithi123 ceph-mon[23709]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-21T10:39:36.822 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:36 smithi123 ceph-mon[23709]: Reconfiguring daemon grafana.smithi123 on smithi123 2024-09-21T10:39:37.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:36 smithi175 ceph-mon[25806]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:39:37.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:36 smithi175 ceph-mon[25806]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:39:37.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:36 smithi175 ceph-mon[25806]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-21T10:39:37.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:36 smithi175 ceph-mon[25806]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-21T10:39:37.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:36 smithi175 ceph-mon[25806]: Reconfiguring daemon grafana.smithi123 on smithi123 2024-09-21T10:39:37.802 INFO:teuthology.orchestra.run.smithi123.stdout: 5 76993 11.43 MB/sec execute 38 sec latency 55.268 ms 2024-09-21T10:39:37.840 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:37 smithi123 ceph-mon[23709]: pgmap v20: 97 pgs: 97 active+clean; 335 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.1 MiB/s wr, 494 op/s 2024-09-21T10:39:38.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:37 smithi175 ceph-mon[25806]: pgmap v20: 97 pgs: 97 active+clean; 335 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.1 MiB/s wr, 494 op/s 2024-09-21T10:39:38.803 INFO:teuthology.orchestra.run.smithi123.stdout: 5 77631 11.84 MB/sec execute 39 sec latency 61.885 ms 2024-09-21T10:39:39.803 INFO:teuthology.orchestra.run.smithi123.stdout: 5 78161 12.11 MB/sec execute 40 sec latency 53.615 ms 2024-09-21T10:39:39.869 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:39 smithi123 ceph-mon[23709]: pgmap v21: 97 pgs: 97 active+clean; 332 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.6 MiB/s wr, 654 op/s 2024-09-21T10:39:40.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:39 smithi175 ceph-mon[25806]: pgmap v21: 97 pgs: 97 active+clean; 332 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.6 MiB/s wr, 654 op/s 2024-09-21T10:39:40.803 INFO:teuthology.orchestra.run.smithi123.stdout: 5 78419 12.00 MB/sec execute 41 sec latency 49.986 ms 2024-09-21T10:39:41.318 INFO:teuthology.orchestra.run.smithi123.stdout:true 2024-09-21T10:39:41.803 INFO:teuthology.orchestra.run.smithi123.stdout: 5 78853 11.79 MB/sec execute 42 sec latency 36.901 ms 2024-09-21T10:39:41.911 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:41 smithi123 ceph-mon[23709]: pgmap v22: 97 pgs: 97 active+clean; 331 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6.6 KiB/s rd, 6.6 MiB/s wr, 580 op/s 2024-09-21T10:39:41.935 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:41 smithi175 ceph-mon[25806]: pgmap v22: 97 pgs: 97 active+clean; 331 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6.6 KiB/s rd, 6.6 MiB/s wr, 580 op/s 2024-09-21T10:39:42.441 INFO:teuthology.orchestra.run.smithi123.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T10:39:42.442 INFO:teuthology.orchestra.run.smithi123.stdout:alertmanager.smithi123 smithi123 *:9093,9094 starting - - - - 2024-09-21T10:39:42.442 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi123 smithi123 running (9m) 28s ago 9m 6405k - 17.2.0 e1d6a67b021e c459f87d5b50 2024-09-21T10:39:42.442 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi175 smithi175 running (8m) 31s ago 8m 6405k - 17.2.0 e1d6a67b021e e35e422d5583 2024-09-21T10:39:42.442 INFO:teuthology.orchestra.run.smithi123.stdout:grafana.smithi123 smithi123 *:3000 running (7m) 28s ago 9m 53.6M - dad864ee21e9 86ee7306dede 2024-09-21T10:39:42.442 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi123.uqoyow smithi123 running (3m) 28s ago 3m 40.4M - 17.2.0 e1d6a67b021e f0a21eaac266 2024-09-21T10:39:42.442 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi175.jrlevw smithi175 running (3m) 31s ago 3m 12.0M - 17.2.0 e1d6a67b021e b21b16f4bc50 2024-09-21T10:39:42.442 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi123.stnuit smithi123 *:9283 running (10m) 28s ago 10m 238M - 17.2.0 e1d6a67b021e 417fa41b561b 2024-09-21T10:39:42.442 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi175.txvwrq smithi175 *:8443,9283 running (55s) 31s ago 8m 498M - 19.3.0-5103-g5c1cdc3c 8f4d013db793 b0be7f732c0b 2024-09-21T10:39:42.442 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi123 smithi123 running (10m) 28s ago 10m 51.1M 2048M 17.2.0 e1d6a67b021e 6ad9a8bc53b7 2024-09-21T10:39:42.442 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi175 smithi175 running (8m) 31s ago 8m 43.1M 2048M 17.2.0 e1d6a67b021e 753f564c8cf5 2024-09-21T10:39:42.442 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.0.0.smithi123.ddatpr smithi123 *:2049 running (2m) 28s ago 2m 252M - 4.0 e1d6a67b021e 36f8cb2281eb 2024-09-21T10:39:42.442 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.1.0.smithi175.wpawer smithi175 *:2049 running (2m) 31s ago 2m 77.9M - 4.0 e1d6a67b021e 2200aeed9200 2024-09-21T10:39:42.442 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi123 smithi123 *:9100 running (9m) 28s ago 9m 18.4M - 1.3.1 1dbe0e931976 875d5b2d6936 2024-09-21T10:39:42.443 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi175 smithi175 *:9100 running (8m) 31s ago 8m 17.0M - 1.3.1 1dbe0e931976 92ddc27e1ca2 2024-09-21T10:39:42.443 INFO:teuthology.orchestra.run.smithi123.stdout:osd.0 smithi123 running (7m) 28s ago 7m 277M 1936M 17.2.0 e1d6a67b021e a94edf88c456 2024-09-21T10:39:42.443 INFO:teuthology.orchestra.run.smithi123.stdout:osd.1 smithi123 running (7m) 28s ago 7m 217M 1936M 17.2.0 e1d6a67b021e 59aee515b0f8 2024-09-21T10:39:42.443 INFO:teuthology.orchestra.run.smithi123.stdout:osd.2 smithi123 running (6m) 28s ago 6m 180M 1936M 17.2.0 e1d6a67b021e 7db93f8ce5b5 2024-09-21T10:39:42.443 INFO:teuthology.orchestra.run.smithi123.stdout:osd.3 smithi123 running (6m) 28s ago 6m 228M 1936M 17.2.0 e1d6a67b021e c1672dea7b1f 2024-09-21T10:39:42.443 INFO:teuthology.orchestra.run.smithi123.stdout:osd.4 smithi175 running (5m) 31s ago 5m 237M 2704M 17.2.0 e1d6a67b021e 48fe728c259e 2024-09-21T10:39:42.443 INFO:teuthology.orchestra.run.smithi123.stdout:osd.5 smithi175 running (5m) 31s ago 5m 280M 2704M 17.2.0 e1d6a67b021e e270d5a21b14 2024-09-21T10:39:42.443 INFO:teuthology.orchestra.run.smithi123.stdout:osd.6 smithi175 running (5m) 31s ago 5m 251M 2704M 17.2.0 e1d6a67b021e 9bcbf018eb20 2024-09-21T10:39:42.443 INFO:teuthology.orchestra.run.smithi123.stdout:osd.7 smithi175 running (4m) 31s ago 4m 351M 2704M 17.2.0 e1d6a67b021e e9b8f1f90994 2024-09-21T10:39:42.443 INFO:teuthology.orchestra.run.smithi123.stdout:prometheus.smithi123 smithi123 *:9095 running (8m) 28s ago 8m 64.1M - 2.33.4 514e6a882f6e 2beddd4ece2e 2024-09-21T10:39:42.803 INFO:teuthology.orchestra.run.smithi123.stdout: 5 79152 11.67 MB/sec execute 43 sec latency 43.807 ms 2024-09-21T10:39:42.933 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:42 smithi123 ceph-mon[23709]: from='client.14726 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:39:42.933 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:42 smithi123 ceph-mon[23709]: from='client.24469 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:39:42.933 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:42 smithi123 ceph-mon[23709]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:39:42.933 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:42 smithi123 ceph-mon[23709]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:39:43.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:42 smithi175 ceph-mon[25806]: from='client.14726 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:39:43.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:42 smithi175 ceph-mon[25806]: from='client.24469 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:39:43.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:42 smithi175 ceph-mon[25806]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:39:43.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:42 smithi175 ceph-mon[25806]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:39:43.162 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-09-21T10:39:43.162 INFO:teuthology.orchestra.run.smithi123.stdout: "mon": { 2024-09-21T10:39:43.162 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T10:39:43.162 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:39:43.162 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr": { 2024-09-21T10:39:43.162 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-21T10:39:43.162 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev)": 1 2024-09-21T10:39:43.162 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:39:43.162 INFO:teuthology.orchestra.run.smithi123.stdout: "osd": { 2024-09-21T10:39:43.162 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-21T10:39:43.162 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:39:43.163 INFO:teuthology.orchestra.run.smithi123.stdout: "mds": { 2024-09-21T10:39:43.163 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T10:39:43.163 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:39:43.163 INFO:teuthology.orchestra.run.smithi123.stdout: "overall": { 2024-09-21T10:39:43.163 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-09-21T10:39:43.163 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev)": 1 2024-09-21T10:39:43.163 INFO:teuthology.orchestra.run.smithi123.stdout: } 2024-09-21T10:39:43.163 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-09-21T10:39:43.698 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:43 smithi123 ceph-mon[23709]: from='client.14734 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:39:43.698 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:43 smithi123 ceph-mon[23709]: Reconfiguring node-exporter.smithi123 (dependencies changed)... 2024-09-21T10:39:43.699 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:43 smithi123 ceph-mon[23709]: Deploying daemon node-exporter.smithi123 on smithi123 2024-09-21T10:39:43.699 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:43 smithi123 ceph-mon[23709]: pgmap v23: 97 pgs: 97 active+clean; 331 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6.6 KiB/s rd, 5.4 MiB/s wr, 489 op/s 2024-09-21T10:39:43.699 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:43 smithi123 ceph-mon[23709]: from='client.? 172.21.15.123:0/1739427814' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T10:39:43.728 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-09-21T10:39:43.728 INFO:teuthology.orchestra.run.smithi123.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472", 2024-09-21T10:39:43.728 INFO:teuthology.orchestra.run.smithi123.stdout: "in_progress": true, 2024-09-21T10:39:43.728 INFO:teuthology.orchestra.run.smithi123.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T10:39:43.728 INFO:teuthology.orchestra.run.smithi123.stdout: "services_complete": [], 2024-09-21T10:39:43.728 INFO:teuthology.orchestra.run.smithi123.stdout: "progress": "1/23 daemons upgraded", 2024-09-21T10:39:43.728 INFO:teuthology.orchestra.run.smithi123.stdout: "message": "", 2024-09-21T10:39:43.728 INFO:teuthology.orchestra.run.smithi123.stdout: "is_paused": false 2024-09-21T10:39:43.728 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-09-21T10:39:43.803 INFO:teuthology.orchestra.run.smithi123.stdout: 5 79512 11.71 MB/sec execute 44 sec latency 53.395 ms 2024-09-21T10:39:44.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:43 smithi175 ceph-mon[25806]: from='client.14734 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:39:44.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:43 smithi175 ceph-mon[25806]: Reconfiguring node-exporter.smithi123 (dependencies changed)... 2024-09-21T10:39:44.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:43 smithi175 ceph-mon[25806]: Deploying daemon node-exporter.smithi123 on smithi123 2024-09-21T10:39:44.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:43 smithi175 ceph-mon[25806]: pgmap v23: 97 pgs: 97 active+clean; 331 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6.6 KiB/s rd, 5.4 MiB/s wr, 489 op/s 2024-09-21T10:39:44.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:43 smithi175 ceph-mon[25806]: from='client.? 172.21.15.123:0/1739427814' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T10:39:44.365 INFO:teuthology.orchestra.run.smithi123.stdout:HEALTH_OK 2024-09-21T10:39:44.706 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:44 smithi123 ceph-mon[23709]: from='client.14742 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:39:44.706 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:44 smithi123 ceph-mon[23709]: from='client.? 172.21.15.123:0/1887360416' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T10:39:44.803 INFO:teuthology.orchestra.run.smithi123.stdout: 5 80006 11.56 MB/sec execute 45 sec latency 34.614 ms 2024-09-21T10:39:45.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:44 smithi175 ceph-mon[25806]: from='client.14742 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:39:45.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:44 smithi175 ceph-mon[25806]: from='client.? 172.21.15.123:0/1887360416' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T10:39:45.726 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:45 smithi123 ceph-mon[23709]: pgmap v24: 97 pgs: 97 active+clean; 331 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.9 MiB/s wr, 733 op/s 2024-09-21T10:39:45.803 INFO:teuthology.orchestra.run.smithi123.stdout: 5 80487 11.47 MB/sec execute 46 sec latency 50.452 ms 2024-09-21T10:39:46.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:45 smithi175 ceph-mon[25806]: pgmap v24: 97 pgs: 97 active+clean; 331 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.9 MiB/s wr, 733 op/s 2024-09-21T10:39:46.803 INFO:teuthology.orchestra.run.smithi123.stdout: 5 81110 11.82 MB/sec execute 47 sec latency 30.133 ms 2024-09-21T10:39:47.803 INFO:teuthology.orchestra.run.smithi123.stdout: 5 81686 12.05 MB/sec execute 48 sec latency 49.504 ms 2024-09-21T10:39:47.994 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:47 smithi123 ceph-mon[23709]: pgmap v25: 97 pgs: 97 active+clean; 331 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.3 MiB/s wr, 524 op/s 2024-09-21T10:39:48.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:47 smithi175 ceph-mon[25806]: pgmap v25: 97 pgs: 97 active+clean; 331 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.3 MiB/s wr, 524 op/s 2024-09-21T10:39:48.803 INFO:teuthology.orchestra.run.smithi123.stdout: 5 82074 12.01 MB/sec execute 49 sec latency 62.001 ms 2024-09-21T10:39:49.803 INFO:teuthology.orchestra.run.smithi123.stdout: 5 82502 11.83 MB/sec execute 50 sec latency 43.300 ms 2024-09-21T10:39:50.015 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:49 smithi123 ceph-mon[23709]: pgmap v26: 97 pgs: 97 active+clean; 338 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.0 MiB/s wr, 703 op/s 2024-09-21T10:39:50.015 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:49 smithi123 ceph-mon[23709]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T10:39:50.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:49 smithi175 ceph-mon[25806]: pgmap v26: 97 pgs: 97 active+clean; 338 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.0 MiB/s wr, 703 op/s 2024-09-21T10:39:50.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:49 smithi175 ceph-mon[25806]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T10:39:50.803 INFO:teuthology.orchestra.run.smithi123.stdout: 5 82700 11.70 MB/sec execute 51 sec latency 71.929 ms 2024-09-21T10:39:51.803 INFO:teuthology.orchestra.run.smithi123.stdout: 5 83160 11.76 MB/sec execute 52 sec latency 41.058 ms 2024-09-21T10:39:52.071 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:51 smithi123 ceph-mon[23709]: pgmap v27: 97 pgs: 97 active+clean; 338 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 7.0 MiB/s wr, 651 op/s 2024-09-21T10:39:52.071 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:51 smithi123 ceph-mon[23709]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:39:52.071 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:51 smithi123 ceph-mon[23709]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:39:52.071 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:51 smithi123 ceph-mon[23709]: Reconfiguring prometheus.smithi123 (dependencies changed)... 2024-09-21T10:39:52.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:51 smithi175 ceph-mon[25806]: pgmap v27: 97 pgs: 97 active+clean; 338 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 7.0 MiB/s wr, 651 op/s 2024-09-21T10:39:52.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:51 smithi175 ceph-mon[25806]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:39:52.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:51 smithi175 ceph-mon[25806]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:39:52.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:51 smithi175 ceph-mon[25806]: Reconfiguring prometheus.smithi123 (dependencies changed)... 2024-09-21T10:39:52.803 INFO:teuthology.orchestra.run.smithi123.stdout: 5 83698 11.65 MB/sec execute 53 sec latency 51.249 ms 2024-09-21T10:39:53.105 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:52 smithi123 ceph-mon[23709]: Deploying daemon prometheus.smithi123 on smithi123 2024-09-21T10:39:53.329 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:52 smithi175 ceph-mon[25806]: Deploying daemon prometheus.smithi123 on smithi123 2024-09-21T10:39:53.803 INFO:teuthology.orchestra.run.smithi123.stdout: 5 84059 11.57 MB/sec execute 54 sec latency 104.602 ms 2024-09-21T10:39:54.117 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:53 smithi123 ceph-mon[23709]: pgmap v28: 97 pgs: 97 active+clean; 338 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.8 MiB/s wr, 530 op/s 2024-09-21T10:39:54.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:53 smithi175 ceph-mon[25806]: pgmap v28: 97 pgs: 97 active+clean; 338 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.8 MiB/s wr, 530 op/s 2024-09-21T10:39:54.803 INFO:teuthology.orchestra.run.smithi123.stdout: 5 84695 11.88 MB/sec execute 55 sec latency 75.988 ms 2024-09-21T10:39:55.803 INFO:teuthology.orchestra.run.smithi123.stdout: 5 85284 12.07 MB/sec execute 56 sec latency 44.273 ms 2024-09-21T10:39:56.184 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:55 smithi123 ceph-mon[23709]: pgmap v29: 97 pgs: 97 active+clean; 326 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 7.7 MiB/s wr, 780 op/s 2024-09-21T10:39:56.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:55 smithi175 ceph-mon[25806]: pgmap v29: 97 pgs: 97 active+clean; 326 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 7.7 MiB/s wr, 780 op/s 2024-09-21T10:39:56.803 INFO:teuthology.orchestra.run.smithi123.stdout: 5 85621 12.00 MB/sec execute 57 sec latency 114.487 ms 2024-09-21T10:39:57.803 INFO:teuthology.orchestra.run.smithi123.stdout: 5 86036 11.86 MB/sec execute 58 sec latency 43.354 ms 2024-09-21T10:39:58.199 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:57 smithi123 ceph-mon[23709]: pgmap v30: 97 pgs: 97 active+clean; 326 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.2 MiB/s wr, 535 op/s 2024-09-21T10:39:58.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:57 smithi175 ceph-mon[25806]: pgmap v30: 97 pgs: 97 active+clean; 326 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.2 MiB/s wr, 535 op/s 2024-09-21T10:39:58.803 INFO:teuthology.orchestra.run.smithi123.stdout: 5 86395 11.85 MB/sec execute 59 sec latency 27.233 ms 2024-09-21T10:39:59.803 INFO:teuthology.orchestra.run.smithi123.stdout: 5 86813 11.79 MB/sec execute 60 sec latency 47.240 ms 2024-09-21T10:40:00.008 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:39:59 smithi123 ceph-mon[23709]: pgmap v31: 97 pgs: 97 active+clean; 344 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 8.3 MiB/s wr, 806 op/s 2024-09-21T10:40:00.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:39:59 smithi175 ceph-mon[25806]: pgmap v31: 97 pgs: 97 active+clean; 344 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 8.3 MiB/s wr, 806 op/s 2024-09-21T10:40:00.803 INFO:teuthology.orchestra.run.smithi123.stdout: 5 87449 11.75 MB/sec execute 61 sec latency 34.839 ms 2024-09-21T10:40:01.020 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:00 smithi123 ceph-mon[23709]: overall HEALTH_OK 2024-09-21T10:40:01.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:00 smithi175 ceph-mon[25806]: overall HEALTH_OK 2024-09-21T10:40:01.804 INFO:teuthology.orchestra.run.smithi123.stdout: 5 88151 12.00 MB/sec execute 62 sec latency 65.869 ms 2024-09-21T10:40:02.035 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:01 smithi123 ceph-mon[23709]: pgmap v32: 97 pgs: 97 active+clean; 345 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 6.8 MiB/s wr, 638 op/s 2024-09-21T10:40:02.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:01 smithi175 ceph-mon[25806]: pgmap v32: 97 pgs: 97 active+clean; 345 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 6.8 MiB/s wr, 638 op/s 2024-09-21T10:40:02.804 INFO:teuthology.orchestra.run.smithi123.stdout: 5 88850 12.25 MB/sec execute 63 sec latency 22.924 ms 2024-09-21T10:40:03.804 INFO:teuthology.orchestra.run.smithi123.stdout: 5 89191 12.18 MB/sec execute 64 sec latency 46.435 ms 2024-09-21T10:40:04.100 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:03 smithi123 ceph-mon[23709]: pgmap v33: 97 pgs: 97 active+clean; 345 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.3 MiB/s wr, 531 op/s 2024-09-21T10:40:04.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:03 smithi175 ceph-mon[25806]: pgmap v33: 97 pgs: 97 active+clean; 345 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.3 MiB/s wr, 531 op/s 2024-09-21T10:40:04.804 INFO:teuthology.orchestra.run.smithi123.stdout: 5 89503 12.05 MB/sec execute 65 sec latency 37.905 ms 2024-09-21T10:40:05.132 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:04 smithi123 ceph-mon[23709]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T10:40:05.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:04 smithi175 ceph-mon[25806]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T10:40:05.804 INFO:teuthology.orchestra.run.smithi123.stdout: 5 89798 11.96 MB/sec execute 66 sec latency 58.217 ms 2024-09-21T10:40:06.153 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:06 smithi123 ceph-mon[23709]: pgmap v34: 97 pgs: 97 active+clean; 340 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.7 MiB/s wr, 854 op/s 2024-09-21T10:40:06.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:06 smithi175 ceph-mon[25806]: pgmap v34: 97 pgs: 97 active+clean; 340 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.7 MiB/s wr, 854 op/s 2024-09-21T10:40:06.804 INFO:teuthology.orchestra.run.smithi123.stdout: 5 90095 11.87 MB/sec execute 67 sec latency 52.162 ms 2024-09-21T10:40:07.804 INFO:teuthology.orchestra.run.smithi123.stdout: 5 90473 11.83 MB/sec execute 68 sec latency 88.102 ms 2024-09-21T10:40:08.238 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:08 smithi123 ceph-mon[23709]: pgmap v35: 97 pgs: 97 active+clean; 340 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.4 KiB/s rd, 6.8 MiB/s wr, 605 op/s 2024-09-21T10:40:08.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:08 smithi175 ceph-mon[25806]: pgmap v35: 97 pgs: 97 active+clean; 340 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.4 KiB/s rd, 6.8 MiB/s wr, 605 op/s 2024-09-21T10:40:08.804 INFO:teuthology.orchestra.run.smithi123.stdout: 5 90793 11.74 MB/sec execute 69 sec latency 70.842 ms 2024-09-21T10:40:09.804 INFO:teuthology.orchestra.run.smithi123.stdout: 5 91171 11.69 MB/sec execute 70 sec latency 49.898 ms 2024-09-21T10:40:10.057 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:10 smithi123 ceph-mon[23709]: pgmap v36: 97 pgs: 97 active+clean; 325 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.3 MiB/s wr, 811 op/s 2024-09-21T10:40:10.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:10 smithi175 ceph-mon[25806]: pgmap v36: 97 pgs: 97 active+clean; 325 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.3 MiB/s wr, 811 op/s 2024-09-21T10:40:10.804 INFO:teuthology.orchestra.run.smithi123.stdout: 5 91468 11.67 MB/sec execute 71 sec latency 70.895 ms 2024-09-21T10:40:11.804 INFO:teuthology.orchestra.run.smithi123.stdout: 5 91835 11.77 MB/sec execute 72 sec latency 46.328 ms 2024-09-21T10:40:11.849 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:11 smithi123 ceph-mon[23709]: pgmap v37: 97 pgs: 97 active+clean; 325 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.2 MiB/s wr, 540 op/s 2024-09-21T10:40:12.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:11 smithi175 ceph-mon[25806]: pgmap v37: 97 pgs: 97 active+clean; 325 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.2 MiB/s wr, 540 op/s 2024-09-21T10:40:12.804 INFO:teuthology.orchestra.run.smithi123.stdout: 5 92119 11.76 MB/sec execute 73 sec latency 62.231 ms 2024-09-21T10:40:13.370 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:13 smithi123 ceph-mon[23709]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:40:13.370 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:13 smithi123 ceph-mon[23709]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:40:13.370 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:13 smithi123 ceph-mon[23709]: Reconfiguring node-exporter.smithi175 (dependencies changed)... 2024-09-21T10:40:13.371 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:13 smithi123 ceph-mon[23709]: Deploying daemon node-exporter.smithi175 on smithi175 2024-09-21T10:40:13.371 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:13 smithi123 ceph-mon[23709]: pgmap v38: 97 pgs: 97 active+clean; 325 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.0 MiB/s wr, 529 op/s 2024-09-21T10:40:13.469 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:13 smithi175 ceph-mon[25806]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:40:13.469 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:13 smithi175 ceph-mon[25806]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:40:13.470 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:13 smithi175 ceph-mon[25806]: Reconfiguring node-exporter.smithi175 (dependencies changed)... 2024-09-21T10:40:13.470 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:13 smithi175 ceph-mon[25806]: Deploying daemon node-exporter.smithi175 on smithi175 2024-09-21T10:40:13.470 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:13 smithi175 ceph-mon[25806]: pgmap v38: 97 pgs: 97 active+clean; 325 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.0 MiB/s wr, 529 op/s 2024-09-21T10:40:13.811 INFO:teuthology.orchestra.run.smithi123.stdout: 5 92434 11.73 MB/sec execute 74 sec latency 60.728 ms 2024-09-21T10:40:14.811 INFO:teuthology.orchestra.run.smithi123.stdout: 5 92830 11.68 MB/sec execute 75 sec latency 43.202 ms 2024-09-21T10:40:14.921 INFO:teuthology.orchestra.run.smithi123.stdout:true 2024-09-21T10:40:15.811 INFO:teuthology.orchestra.run.smithi123.stdout: 5 93313 11.67 MB/sec execute 76 sec latency 40.120 ms 2024-09-21T10:40:15.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:15 smithi123 ceph-mon[23709]: pgmap v39: 97 pgs: 97 active+clean; 334 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.4 MiB/s wr, 716 op/s 2024-09-21T10:40:15.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:15 smithi123 ceph-mon[23709]: from='client.14750 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:40:16.072 INFO:teuthology.orchestra.run.smithi123.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T10:40:16.073 INFO:teuthology.orchestra.run.smithi123.stdout:alertmanager.smithi123 smithi123 *:9093,9094 starting - - - - 2024-09-21T10:40:16.073 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi123 smithi123 running (9m) 61s ago 9m 6405k - 17.2.0 e1d6a67b021e c459f87d5b50 2024-09-21T10:40:16.073 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi175 smithi175 running (9m) 64s ago 9m 6405k - 17.2.0 e1d6a67b021e e35e422d5583 2024-09-21T10:40:16.073 INFO:teuthology.orchestra.run.smithi123.stdout:grafana.smithi123 smithi123 *:3000 starting - - - - 2024-09-21T10:40:16.073 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi123.uqoyow smithi123 running (3m) 61s ago 3m 40.4M - 17.2.0 e1d6a67b021e f0a21eaac266 2024-09-21T10:40:16.073 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi175.jrlevw smithi175 running (3m) 64s ago 3m 12.0M - 17.2.0 e1d6a67b021e b21b16f4bc50 2024-09-21T10:40:16.073 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi123.stnuit smithi123 *:9283 running (11m) 61s ago 11m 238M - 17.2.0 e1d6a67b021e 417fa41b561b 2024-09-21T10:40:16.073 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi175.txvwrq smithi175 *:8443,9283 running (89s) 64s ago 9m 498M - 19.3.0-5103-g5c1cdc3c 8f4d013db793 b0be7f732c0b 2024-09-21T10:40:16.073 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi123 smithi123 running (11m) 61s ago 11m 51.1M 2048M 17.2.0 e1d6a67b021e 6ad9a8bc53b7 2024-09-21T10:40:16.073 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi175 smithi175 running (9m) 64s ago 9m 43.1M 2048M 17.2.0 e1d6a67b021e 753f564c8cf5 2024-09-21T10:40:16.073 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.0.0.smithi123.ddatpr smithi123 *:2049 running (3m) 61s ago 3m 252M - 4.0 e1d6a67b021e 36f8cb2281eb 2024-09-21T10:40:16.073 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.1.0.smithi175.wpawer smithi175 *:2049 running (3m) 64s ago 3m 77.9M - 4.0 e1d6a67b021e 2200aeed9200 2024-09-21T10:40:16.074 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi123 smithi123 *:9100 starting - - - - 2024-09-21T10:40:16.074 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi175 smithi175 *:9100 running (8m) 64s ago 8m 17.0M - 1.3.1 1dbe0e931976 92ddc27e1ca2 2024-09-21T10:40:16.074 INFO:teuthology.orchestra.run.smithi123.stdout:osd.0 smithi123 running (8m) 61s ago 8m 277M 1936M 17.2.0 e1d6a67b021e a94edf88c456 2024-09-21T10:40:16.074 INFO:teuthology.orchestra.run.smithi123.stdout:osd.1 smithi123 running (7m) 61s ago 7m 217M 1936M 17.2.0 e1d6a67b021e 59aee515b0f8 2024-09-21T10:40:16.074 INFO:teuthology.orchestra.run.smithi123.stdout:osd.2 smithi123 running (7m) 61s ago 7m 180M 1936M 17.2.0 e1d6a67b021e 7db93f8ce5b5 2024-09-21T10:40:16.074 INFO:teuthology.orchestra.run.smithi123.stdout:osd.3 smithi123 running (6m) 61s ago 6m 228M 1936M 17.2.0 e1d6a67b021e c1672dea7b1f 2024-09-21T10:40:16.074 INFO:teuthology.orchestra.run.smithi123.stdout:osd.4 smithi175 running (6m) 64s ago 6m 237M 2704M 17.2.0 e1d6a67b021e 48fe728c259e 2024-09-21T10:40:16.074 INFO:teuthology.orchestra.run.smithi123.stdout:osd.5 smithi175 running (6m) 64s ago 6m 280M 2704M 17.2.0 e1d6a67b021e e270d5a21b14 2024-09-21T10:40:16.074 INFO:teuthology.orchestra.run.smithi123.stdout:osd.6 smithi175 running (5m) 64s ago 5m 251M 2704M 17.2.0 e1d6a67b021e 9bcbf018eb20 2024-09-21T10:40:16.074 INFO:teuthology.orchestra.run.smithi123.stdout:osd.7 smithi175 running (5m) 64s ago 5m 351M 2704M 17.2.0 e1d6a67b021e e9b8f1f90994 2024-09-21T10:40:16.074 INFO:teuthology.orchestra.run.smithi123.stdout:prometheus.smithi123 smithi123 *:9095 starting - - - - 2024-09-21T10:40:16.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:15 smithi175 ceph-mon[25806]: pgmap v39: 97 pgs: 97 active+clean; 334 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.4 MiB/s wr, 716 op/s 2024-09-21T10:40:16.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:15 smithi175 ceph-mon[25806]: from='client.14750 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:40:16.688 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-09-21T10:40:16.688 INFO:teuthology.orchestra.run.smithi123.stdout: "mon": { 2024-09-21T10:40:16.688 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T10:40:16.688 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:40:16.689 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr": { 2024-09-21T10:40:16.689 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-21T10:40:16.689 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev)": 1 2024-09-21T10:40:16.689 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:40:16.689 INFO:teuthology.orchestra.run.smithi123.stdout: "osd": { 2024-09-21T10:40:16.689 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-21T10:40:16.689 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:40:16.689 INFO:teuthology.orchestra.run.smithi123.stdout: "mds": { 2024-09-21T10:40:16.689 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T10:40:16.689 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:40:16.689 INFO:teuthology.orchestra.run.smithi123.stdout: "overall": { 2024-09-21T10:40:16.689 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-09-21T10:40:16.689 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev)": 1 2024-09-21T10:40:16.689 INFO:teuthology.orchestra.run.smithi123.stdout: } 2024-09-21T10:40:16.689 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-09-21T10:40:16.811 INFO:teuthology.orchestra.run.smithi123.stdout: 5 93698 11.57 MB/sec execute 77 sec latency 74.631 ms 2024-09-21T10:40:16.948 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:16 smithi123 ceph-mon[23709]: from='client.24489 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:40:16.948 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:16 smithi123 ceph-mon[23709]: from='client.14758 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:40:16.948 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:16 smithi123 ceph-mon[23709]: from='client.? 172.21.15.123:0/1297015159' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T10:40:17.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:16 smithi175 ceph-mon[25806]: from='client.24489 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:40:17.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:16 smithi175 ceph-mon[25806]: from='client.14758 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:40:17.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:16 smithi175 ceph-mon[25806]: from='client.? 172.21.15.123:0/1297015159' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T10:40:17.229 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-09-21T10:40:17.229 INFO:teuthology.orchestra.run.smithi123.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472", 2024-09-21T10:40:17.229 INFO:teuthology.orchestra.run.smithi123.stdout: "in_progress": true, 2024-09-21T10:40:17.229 INFO:teuthology.orchestra.run.smithi123.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T10:40:17.229 INFO:teuthology.orchestra.run.smithi123.stdout: "services_complete": [], 2024-09-21T10:40:17.229 INFO:teuthology.orchestra.run.smithi123.stdout: "progress": "1/23 daemons upgraded", 2024-09-21T10:40:17.230 INFO:teuthology.orchestra.run.smithi123.stdout: "message": "", 2024-09-21T10:40:17.230 INFO:teuthology.orchestra.run.smithi123.stdout: "is_paused": false 2024-09-21T10:40:17.230 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-09-21T10:40:17.811 INFO:teuthology.orchestra.run.smithi123.stdout: 5 94103 11.56 MB/sec execute 78 sec latency 49.306 ms 2024-09-21T10:40:17.881 INFO:teuthology.orchestra.run.smithi123.stdout:HEALTH_OK 2024-09-21T10:40:17.961 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:17 smithi123 ceph-mon[23709]: pgmap v40: 97 pgs: 97 active+clean; 334 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.6 KiB/s rd, 3.9 MiB/s wr, 392 op/s 2024-09-21T10:40:17.969 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:17 smithi175 ceph-mon[25806]: pgmap v40: 97 pgs: 97 active+clean; 334 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.6 KiB/s rd, 3.9 MiB/s wr, 392 op/s 2024-09-21T10:40:18.812 INFO:teuthology.orchestra.run.smithi123.stdout: 5 94597 11.54 MB/sec execute 79 sec latency 54.447 ms 2024-09-21T10:40:18.987 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:18 smithi123 ceph-mon[23709]: from='client.24497 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:40:18.987 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:18 smithi123 ceph-mon[23709]: from='client.? 172.21.15.123:0/2999278202' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T10:40:18.988 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:18 smithi123 ceph-mon[23709]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:40:18.988 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:18 smithi123 ceph-mon[23709]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:40:18.988 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:18 smithi123 ceph-mon[23709]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T10:40:18.988 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:18 smithi123 ceph-mon[23709]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi123.front.sepia.ceph.com:9093"}]: dispatch 2024-09-21T10:40:18.988 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:18 smithi123 ceph-mon[23709]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:40:18.988 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:18 smithi123 ceph-mon[23709]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T10:40:18.988 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:18 smithi123 ceph-mon[23709]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi123.front.sepia.ceph.com:3000"}]: dispatch 2024-09-21T10:40:18.988 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:18 smithi123 ceph-mon[23709]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:40:18.988 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:18 smithi123 ceph-mon[23709]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T10:40:18.988 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:18 smithi123 ceph-mon[23709]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi123.front.sepia.ceph.com:9095"}]: dispatch 2024-09-21T10:40:18.988 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:18 smithi123 ceph-mon[23709]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:40:18.988 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:18 smithi123 ceph-mon[23709]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T10:40:18.988 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:18 smithi123 ceph-mon[23709]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:40:19.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:18 smithi175 ceph-mon[25806]: from='client.24497 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:40:19.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:18 smithi175 ceph-mon[25806]: from='client.? 172.21.15.123:0/2999278202' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T10:40:19.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:18 smithi175 ceph-mon[25806]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:40:19.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:18 smithi175 ceph-mon[25806]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:40:19.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:18 smithi175 ceph-mon[25806]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T10:40:19.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:18 smithi175 ceph-mon[25806]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi123.front.sepia.ceph.com:9093"}]: dispatch 2024-09-21T10:40:19.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:18 smithi175 ceph-mon[25806]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:40:19.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:18 smithi175 ceph-mon[25806]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T10:40:19.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:18 smithi175 ceph-mon[25806]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi123.front.sepia.ceph.com:3000"}]: dispatch 2024-09-21T10:40:19.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:18 smithi175 ceph-mon[25806]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:40:19.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:18 smithi175 ceph-mon[25806]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T10:40:19.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:18 smithi175 ceph-mon[25806]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi123.front.sepia.ceph.com:9095"}]: dispatch 2024-09-21T10:40:19.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:18 smithi175 ceph-mon[25806]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:40:19.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:18 smithi175 ceph-mon[25806]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T10:40:19.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:18 smithi175 ceph-mon[25806]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:40:19.812 INFO:teuthology.orchestra.run.smithi123.stdout: 5 95072 11.58 MB/sec execute 80 sec latency 30.897 ms 2024-09-21T10:40:19.991 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:19 smithi123 ceph-mon[23709]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T10:40:19.991 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:19 smithi123 ceph-mon[23709]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi123.front.sepia.ceph.com:9093"}]: dispatch 2024-09-21T10:40:19.991 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:19 smithi123 ceph-mon[23709]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T10:40:19.991 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:19 smithi123 ceph-mon[23709]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi123.front.sepia.ceph.com:3000"}]: dispatch 2024-09-21T10:40:19.991 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:19 smithi123 ceph-mon[23709]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T10:40:19.992 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:19 smithi123 ceph-mon[23709]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi123.front.sepia.ceph.com:9095"}]: dispatch 2024-09-21T10:40:19.992 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:19 smithi123 ceph-mon[23709]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T10:40:19.992 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:19 smithi123 ceph-mon[23709]: Upgrade: Need to upgrade myself (mgr.smithi175.txvwrq) 2024-09-21T10:40:19.992 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:19 smithi123 ceph-mon[23709]: Upgrade: Need to upgrade myself (mgr.smithi175.txvwrq) 2024-09-21T10:40:19.992 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:19 smithi123 ceph-mon[23709]: pgmap v41: 97 pgs: 97 active+clean; 335 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 5.9 MiB/s wr, 643 op/s 2024-09-21T10:40:19.992 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:19 smithi123 ceph-mon[23709]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T10:40:19.992 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:19 smithi123 ceph-mon[23709]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T10:40:19.992 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:19 smithi123 ceph-mon[23709]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T10:40:19.992 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:19 smithi123 ceph-mon[23709]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T10:40:20.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:19 smithi175 ceph-mon[25806]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T10:40:20.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:19 smithi175 ceph-mon[25806]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi123.front.sepia.ceph.com:9093"}]: dispatch 2024-09-21T10:40:20.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:19 smithi175 ceph-mon[25806]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T10:40:20.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:19 smithi175 ceph-mon[25806]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi123.front.sepia.ceph.com:3000"}]: dispatch 2024-09-21T10:40:20.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:19 smithi175 ceph-mon[25806]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T10:40:20.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:19 smithi175 ceph-mon[25806]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi123.front.sepia.ceph.com:9095"}]: dispatch 2024-09-21T10:40:20.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:19 smithi175 ceph-mon[25806]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T10:40:20.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:19 smithi175 ceph-mon[25806]: Upgrade: Need to upgrade myself (mgr.smithi175.txvwrq) 2024-09-21T10:40:20.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:19 smithi175 ceph-mon[25806]: Upgrade: Need to upgrade myself (mgr.smithi175.txvwrq) 2024-09-21T10:40:20.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:19 smithi175 ceph-mon[25806]: pgmap v41: 97 pgs: 97 active+clean; 335 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 5.9 MiB/s wr, 643 op/s 2024-09-21T10:40:20.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:19 smithi175 ceph-mon[25806]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T10:40:20.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:19 smithi175 ceph-mon[25806]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T10:40:20.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:19 smithi175 ceph-mon[25806]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T10:40:20.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:19 smithi175 ceph-mon[25806]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T10:40:20.812 INFO:teuthology.orchestra.run.smithi123.stdout: 5 95589 11.73 MB/sec execute 81 sec latency 34.900 ms 2024-09-21T10:40:21.016 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:20 smithi123 ceph-mon[23709]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T10:40:21.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:20 smithi175 ceph-mon[25806]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T10:40:21.812 INFO:teuthology.orchestra.run.smithi123.stdout: 5 95938 11.74 MB/sec execute 82 sec latency 58.530 ms 2024-09-21T10:40:22.046 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:21 smithi123 ceph-mon[23709]: pgmap v42: 97 pgs: 97 active+clean; 328 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.6 MiB/s wr, 454 op/s 2024-09-21T10:40:22.046 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:21 smithi123 ceph-mon[23709]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:40:22.046 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:21 smithi123 ceph-mon[23709]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi123.stnuit", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T10:40:22.046 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:21 smithi123 ceph-mon[23709]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi123.stnuit", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T10:40:22.046 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:21 smithi123 ceph-mon[23709]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T10:40:22.047 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:21 smithi123 ceph-mon[23709]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:40:22.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:21 smithi175 ceph-mon[25806]: pgmap v42: 97 pgs: 97 active+clean; 328 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.6 MiB/s wr, 454 op/s 2024-09-21T10:40:22.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:21 smithi175 ceph-mon[25806]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:40:22.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:21 smithi175 ceph-mon[25806]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi123.stnuit", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T10:40:22.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:21 smithi175 ceph-mon[25806]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi123.stnuit", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T10:40:22.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:21 smithi175 ceph-mon[25806]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T10:40:22.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:21 smithi175 ceph-mon[25806]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:40:22.812 INFO:teuthology.orchestra.run.smithi123.stdout: 5 96258 11.68 MB/sec execute 83 sec latency 67.025 ms 2024-09-21T10:40:23.063 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:22 smithi123 ceph-mon[23709]: Upgrade: Updating mgr.smithi123.stnuit 2024-09-21T10:40:23.063 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:22 smithi123 ceph-mon[23709]: Deploying daemon mgr.smithi123.stnuit on smithi123 2024-09-21T10:40:23.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:22 smithi175 ceph-mon[25806]: Upgrade: Updating mgr.smithi123.stnuit 2024-09-21T10:40:23.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:22 smithi175 ceph-mon[25806]: Deploying daemon mgr.smithi123.stnuit on smithi123 2024-09-21T10:40:23.812 INFO:teuthology.orchestra.run.smithi123.stdout: 5 96590 11.61 MB/sec execute 84 sec latency 57.558 ms 2024-09-21T10:40:24.090 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:23 smithi123 ceph-mon[23709]: pgmap v43: 97 pgs: 97 active+clean; 328 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.6 MiB/s wr, 454 op/s 2024-09-21T10:40:24.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:23 smithi175 ceph-mon[25806]: pgmap v43: 97 pgs: 97 active+clean; 328 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.6 MiB/s wr, 454 op/s 2024-09-21T10:40:24.813 INFO:teuthology.orchestra.run.smithi123.stdout: 5 96898 11.56 MB/sec execute 85 sec latency 75.424 ms 2024-09-21T10:40:25.813 INFO:teuthology.orchestra.run.smithi123.stdout: 5 97259 11.47 MB/sec execute 86 sec latency 37.832 ms 2024-09-21T10:40:25.937 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:25 smithi123 ceph-mon[23709]: pgmap v44: 97 pgs: 97 active+clean; 340 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 9.0 KiB/s rd, 7.7 MiB/s wr, 695 op/s 2024-09-21T10:40:26.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:25 smithi175 ceph-mon[25806]: pgmap v44: 97 pgs: 97 active+clean; 340 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 9.0 KiB/s rd, 7.7 MiB/s wr, 695 op/s 2024-09-21T10:40:26.813 INFO:teuthology.orchestra.run.smithi123.stdout: 5 97643 11.46 MB/sec execute 87 sec latency 35.612 ms 2024-09-21T10:40:27.813 INFO:teuthology.orchestra.run.smithi123.stdout: 5 97989 11.40 MB/sec execute 88 sec latency 57.417 ms 2024-09-21T10:40:27.975 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:27 smithi123 ceph-mon[23709]: pgmap v45: 97 pgs: 97 active+clean; 340 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.3 MiB/s wr, 508 op/s 2024-09-21T10:40:28.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:27 smithi175 ceph-mon[25806]: pgmap v45: 97 pgs: 97 active+clean; 340 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.3 MiB/s wr, 508 op/s 2024-09-21T10:40:28.813 INFO:teuthology.orchestra.run.smithi123.stdout: 5 98419 11.42 MB/sec execute 89 sec latency 110.852 ms 2024-09-21T10:40:29.813 INFO:teuthology.orchestra.run.smithi123.stdout: 5 98892 11.52 MB/sec execute 90 sec latency 84.458 ms 2024-09-21T10:40:30.056 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:29 smithi123 ceph-mon[23709]: pgmap v46: 97 pgs: 97 active+clean; 326 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.4 MiB/s wr, 614 op/s 2024-09-21T10:40:30.056 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:29 smithi123 ceph-mon[23709]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:40:30.056 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:29 smithi123 ceph-mon[23709]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:40:30.056 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:29 smithi123 ceph-mon[23709]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:40:30.057 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:29 smithi123 ceph-mon[23709]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:40:30.057 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:29 smithi123 ceph-mon[23709]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:40:30.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:29 smithi175 ceph-mon[25806]: pgmap v46: 97 pgs: 97 active+clean; 326 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.4 MiB/s wr, 614 op/s 2024-09-21T10:40:30.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:29 smithi175 ceph-mon[25806]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:40:30.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:29 smithi175 ceph-mon[25806]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:40:30.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:29 smithi175 ceph-mon[25806]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:40:30.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:29 smithi175 ceph-mon[25806]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:40:30.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:29 smithi175 ceph-mon[25806]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:40:30.813 INFO:teuthology.orchestra.run.smithi123.stdout: 5 99237 11.54 MB/sec execute 91 sec latency 42.609 ms 2024-09-21T10:40:31.813 INFO:teuthology.orchestra.run.smithi123.stdout: 5 99667 11.57 MB/sec execute 92 sec latency 31.492 ms 2024-09-21T10:40:31.827 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:31 smithi123 ceph-mon[23709]: pgmap v47: 97 pgs: 97 active+clean; 328 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.1 MiB/s wr, 475 op/s 2024-09-21T10:40:32.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:31 smithi175 ceph-mon[25806]: pgmap v47: 97 pgs: 97 active+clean; 328 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.1 MiB/s wr, 475 op/s 2024-09-21T10:40:32.813 INFO:teuthology.orchestra.run.smithi123.stdout: 5 99895 11.48 MB/sec execute 93 sec latency 76.281 ms 2024-09-21T10:40:33.813 INFO:teuthology.orchestra.run.smithi123.stdout: 5 100150 11.39 MB/sec execute 94 sec latency 72.969 ms 2024-09-21T10:40:34.084 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:33 smithi123 ceph-mon[23709]: pgmap v48: 97 pgs: 97 active+clean; 328 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.8 MiB/s wr, 458 op/s 2024-09-21T10:40:34.084 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:33 smithi123 ceph-mon[23709]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:40:34.084 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:33 smithi123 ceph-mon[23709]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:40:34.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:33 smithi175 ceph-mon[25806]: pgmap v48: 97 pgs: 97 active+clean; 328 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.8 MiB/s wr, 458 op/s 2024-09-21T10:40:34.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:33 smithi175 ceph-mon[25806]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:40:34.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:33 smithi175 ceph-mon[25806]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:40:34.813 INFO:teuthology.orchestra.run.smithi123.stdout: 5 100490 11.36 MB/sec execute 95 sec latency 53.992 ms 2024-09-21T10:40:35.095 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:34 smithi123 ceph-mon[23709]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T10:40:35.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:34 smithi175 ceph-mon[25806]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T10:40:35.813 INFO:teuthology.orchestra.run.smithi123.stdout: 5 100882 11.28 MB/sec execute 96 sec latency 31.679 ms 2024-09-21T10:40:36.100 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:35 smithi123 ceph-mon[23709]: pgmap v49: 97 pgs: 97 active+clean; 341 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 9.4 KiB/s rd, 7.4 MiB/s wr, 670 op/s 2024-09-21T10:40:36.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:35 smithi175 ceph-mon[25806]: pgmap v49: 97 pgs: 97 active+clean; 341 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 9.4 KiB/s rd, 7.4 MiB/s wr, 670 op/s 2024-09-21T10:40:36.813 INFO:teuthology.orchestra.run.smithi123.stdout: 5 101239 11.27 MB/sec execute 97 sec latency 45.030 ms 2024-09-21T10:40:37.120 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:36 smithi123 ceph-mon[23709]: Standby manager daemon smithi123.stnuit restarted 2024-09-21T10:40:37.120 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:36 smithi123 ceph-mon[23709]: Standby manager daemon smithi123.stnuit started 2024-09-21T10:40:37.120 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:36 smithi123 ceph-mon[23709]: from='mgr.? 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi123.stnuit/crt"}]: dispatch 2024-09-21T10:40:37.121 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:36 smithi123 ceph-mon[23709]: from='mgr.? 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T10:40:37.121 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:36 smithi123 ceph-mon[23709]: from='mgr.? 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi123.stnuit/key"}]: dispatch 2024-09-21T10:40:37.121 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:36 smithi123 ceph-mon[23709]: from='mgr.? 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T10:40:37.349 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:36 smithi175 ceph-mon[25806]: Standby manager daemon smithi123.stnuit restarted 2024-09-21T10:40:37.349 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:36 smithi175 ceph-mon[25806]: Standby manager daemon smithi123.stnuit started 2024-09-21T10:40:37.349 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:36 smithi175 ceph-mon[25806]: from='mgr.? 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi123.stnuit/crt"}]: dispatch 2024-09-21T10:40:37.349 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:36 smithi175 ceph-mon[25806]: from='mgr.? 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T10:40:37.349 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:36 smithi175 ceph-mon[25806]: from='mgr.? 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi123.stnuit/key"}]: dispatch 2024-09-21T10:40:37.349 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:36 smithi175 ceph-mon[25806]: from='mgr.? 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T10:40:37.814 INFO:teuthology.orchestra.run.smithi123.stdout: 5 101724 11.26 MB/sec execute 98 sec latency 46.977 ms 2024-09-21T10:40:38.127 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:37 smithi123 ceph-mon[23709]: pgmap v50: 97 pgs: 97 active+clean; 341 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.4 MiB/s wr, 429 op/s 2024-09-21T10:40:38.127 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:37 smithi123 ceph-mon[23709]: mgrmap e28: smithi175.txvwrq(active, since 95s), standbys: smithi123.stnuit 2024-09-21T10:40:38.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:37 smithi175 ceph-mon[25806]: pgmap v50: 97 pgs: 97 active+clean; 341 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.4 MiB/s wr, 429 op/s 2024-09-21T10:40:38.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:37 smithi175 ceph-mon[25806]: mgrmap e28: smithi175.txvwrq(active, since 95s), standbys: smithi123.stnuit 2024-09-21T10:40:38.814 INFO:teuthology.orchestra.run.smithi123.stdout: 5 102154 11.28 MB/sec execute 99 sec latency 40.890 ms 2024-09-21T10:40:39.814 INFO:teuthology.orchestra.run.smithi123.stdout: 5 102540 11.36 MB/sec execute 100 sec latency 35.847 ms 2024-09-21T10:40:40.154 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:39 smithi123 ceph-mon[23709]: pgmap v51: 97 pgs: 97 active+clean; 337 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.0 MiB/s wr, 635 op/s 2024-09-21T10:40:40.154 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:39 smithi123 ceph-mon[23709]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:40:40.154 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:39 smithi123 ceph-mon[23709]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:40:40.154 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:39 smithi123 ceph-mon[23709]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:40:40.154 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:39 smithi123 ceph-mon[23709]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:40:40.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:39 smithi175 ceph-mon[25806]: pgmap v51: 97 pgs: 97 active+clean; 337 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.0 MiB/s wr, 635 op/s 2024-09-21T10:40:40.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:39 smithi175 ceph-mon[25806]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:40:40.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:39 smithi175 ceph-mon[25806]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:40:40.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:39 smithi175 ceph-mon[25806]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:40:40.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:39 smithi175 ceph-mon[25806]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:40:40.814 INFO:teuthology.orchestra.run.smithi123.stdout: 5 102983 11.42 MB/sec execute 101 sec latency 61.831 ms 2024-09-21T10:40:41.814 INFO:teuthology.orchestra.run.smithi123.stdout: 5 103313 11.38 MB/sec execute 102 sec latency 41.674 ms 2024-09-21T10:40:41.927 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:41 smithi123 ceph-mon[23709]: pgmap v52: 97 pgs: 97 active+clean; 331 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.6 KiB/s rd, 5.3 MiB/s wr, 565 op/s 2024-09-21T10:40:42.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:41 smithi175 ceph-mon[25806]: pgmap v52: 97 pgs: 97 active+clean; 331 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.6 KiB/s rd, 5.3 MiB/s wr, 565 op/s 2024-09-21T10:40:42.814 INFO:teuthology.orchestra.run.smithi123.stdout: 5 103554 11.31 MB/sec execute 103 sec latency 35.849 ms 2024-09-21T10:40:43.814 INFO:teuthology.orchestra.run.smithi123.stdout: 5 103959 11.26 MB/sec execute 104 sec latency 50.062 ms 2024-09-21T10:40:43.949 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:43 smithi123 ceph-mon[23709]: pgmap v53: 97 pgs: 97 active+clean; 331 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.8 MiB/s wr, 453 op/s 2024-09-21T10:40:44.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:43 smithi175 ceph-mon[25806]: pgmap v53: 97 pgs: 97 active+clean; 331 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.8 MiB/s wr, 453 op/s 2024-09-21T10:40:44.814 INFO:teuthology.orchestra.run.smithi123.stdout: 5 104536 11.28 MB/sec execute 105 sec latency 43.309 ms 2024-09-21T10:40:45.814 INFO:teuthology.orchestra.run.smithi123.stdout: 5 105017 11.26 MB/sec execute 106 sec latency 51.526 ms 2024-09-21T10:40:45.962 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:45 smithi123 ceph-mon[23709]: pgmap v54: 97 pgs: 97 active+clean; 341 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.5 KiB/s rd, 7.2 MiB/s wr, 674 op/s 2024-09-21T10:40:46.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:45 smithi175 ceph-mon[25806]: pgmap v54: 97 pgs: 97 active+clean; 341 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.5 KiB/s rd, 7.2 MiB/s wr, 674 op/s 2024-09-21T10:40:46.814 INFO:teuthology.orchestra.run.smithi123.stdout: 5 105886 11.43 MB/sec execute 107 sec latency 70.528 ms 2024-09-21T10:40:47.494 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:47 smithi123 ceph-mon[23709]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:40:47.494 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:47 smithi123 ceph-mon[23709]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:40:47.494 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:47 smithi123 ceph-mon[23709]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:40:47.494 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:47 smithi123 ceph-mon[23709]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T10:40:47.494 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:47 smithi123 ceph-mon[23709]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:40:47.494 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:47 smithi123 ceph-mon[23709]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:40:47.494 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:47 smithi123 ceph-mon[23709]: pgmap v55: 97 pgs: 97 active+clean; 341 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.6 MiB/s wr, 463 op/s 2024-09-21T10:40:47.495 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:47 smithi123 ceph-mon[23709]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:40:47.495 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:47 smithi123 ceph-mon[23709]: Upgrade: Need to upgrade myself (mgr.smithi175.txvwrq) 2024-09-21T10:40:47.495 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:47 smithi123 ceph-mon[23709]: Upgrade: Need to upgrade myself (mgr.smithi175.txvwrq) 2024-09-21T10:40:47.495 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:47 smithi123 ceph-mon[23709]: Failing over to other MGR 2024-09-21T10:40:47.495 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:47 smithi123 ceph-mon[23709]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "mgr fail", "who": "smithi175.txvwrq"}]: dispatch 2024-09-21T10:40:47.495 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:47 smithi123 ceph-mon[23709]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "mgr fail", "who": "smithi175.txvwrq"}]: dispatch 2024-09-21T10:40:47.495 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:47 smithi123 ceph-mon[23709]: osdmap e75: 8 total, 8 up, 8 in 2024-09-21T10:40:47.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:47 smithi175 ceph-mon[25806]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:40:47.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:47 smithi175 ceph-mon[25806]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:40:47.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:47 smithi175 ceph-mon[25806]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:40:47.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:47 smithi175 ceph-mon[25806]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T10:40:47.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:47 smithi175 ceph-mon[25806]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:40:47.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:47 smithi175 ceph-mon[25806]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' 2024-09-21T10:40:47.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:47 smithi175 ceph-mon[25806]: pgmap v55: 97 pgs: 97 active+clean; 341 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.6 MiB/s wr, 463 op/s 2024-09-21T10:40:47.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:47 smithi175 ceph-mon[25806]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:40:47.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:47 smithi175 ceph-mon[25806]: Upgrade: Need to upgrade myself (mgr.smithi175.txvwrq) 2024-09-21T10:40:47.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:47 smithi175 ceph-mon[25806]: Upgrade: Need to upgrade myself (mgr.smithi175.txvwrq) 2024-09-21T10:40:47.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:47 smithi175 ceph-mon[25806]: Failing over to other MGR 2024-09-21T10:40:47.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:47 smithi175 ceph-mon[25806]: from='mgr.24415 172.21.15.175:0/50299521' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "mgr fail", "who": "smithi175.txvwrq"}]: dispatch 2024-09-21T10:40:47.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:47 smithi175 ceph-mon[25806]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "mgr fail", "who": "smithi175.txvwrq"}]: dispatch 2024-09-21T10:40:47.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:47 smithi175 ceph-mon[25806]: osdmap e75: 8 total, 8 up, 8 in 2024-09-21T10:40:47.814 INFO:teuthology.orchestra.run.smithi123.stdout: 5 106375 11.49 MB/sec execute 108 sec latency 38.322 ms 2024-09-21T10:40:48.498 INFO:teuthology.orchestra.run.smithi123.stdout:true 2024-09-21T10:40:48.527 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:48 smithi123 ceph-mon[23709]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' cmd='[{"prefix": "mgr fail", "who": "smithi175.txvwrq"}]': finished 2024-09-21T10:40:48.527 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:48 smithi123 ceph-mon[23709]: mgrmap e29: smithi123.stnuit(active, starting, since 0.633213s) 2024-09-21T10:40:48.528 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:48 smithi123 ceph-mon[23709]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mon metadata", "id": "smithi123"}]: dispatch 2024-09-21T10:40:48.528 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:48 smithi123 ceph-mon[23709]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mon metadata", "id": "smithi175"}]: dispatch 2024-09-21T10:40:48.528 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:48 smithi123 ceph-mon[23709]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi123.uqoyow"}]: dispatch 2024-09-21T10:40:48.528 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:48 smithi123 ceph-mon[23709]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi175.jrlevw"}]: dispatch 2024-09-21T10:40:48.528 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:48 smithi123 ceph-mon[23709]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mgr metadata", "who": "smithi123.stnuit", "id": "smithi123.stnuit"}]: dispatch 2024-09-21T10:40:48.528 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:48 smithi123 ceph-mon[23709]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T10:40:48.528 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:48 smithi123 ceph-mon[23709]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T10:40:48.528 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:48 smithi123 ceph-mon[23709]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T10:40:48.528 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:48 smithi123 ceph-mon[23709]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T10:40:48.528 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:48 smithi123 ceph-mon[23709]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T10:40:48.528 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:48 smithi123 ceph-mon[23709]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T10:40:48.529 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:48 smithi123 ceph-mon[23709]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T10:40:48.529 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:48 smithi123 ceph-mon[23709]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T10:40:48.529 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:48 smithi123 ceph-mon[23709]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T10:40:48.529 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:48 smithi123 ceph-mon[23709]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T10:40:48.529 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:48 smithi123 ceph-mon[23709]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T10:40:48.529 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:48 smithi123 ceph-mon[23709]: Manager daemon smithi123.stnuit is now available 2024-09-21T10:40:48.529 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:48 smithi123 ceph-mon[23709]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:40:48.529 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:48 smithi123 ceph-mon[23709]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T10:40:48.529 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:48 smithi123 ceph-mon[23709]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T10:40:48.529 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:48 smithi123 ceph-mon[23709]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.stnuit/mirror_snapshot_schedule"}]: dispatch 2024-09-21T10:40:48.687 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:48 smithi175 ceph-mon[25806]: from='mgr.24415 ' entity='mgr.smithi175.txvwrq' cmd='[{"prefix": "mgr fail", "who": "smithi175.txvwrq"}]': finished 2024-09-21T10:40:48.687 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:48 smithi175 ceph-mon[25806]: mgrmap e29: smithi123.stnuit(active, starting, since 0.633213s) 2024-09-21T10:40:48.687 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:48 smithi175 ceph-mon[25806]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mon metadata", "id": "smithi123"}]: dispatch 2024-09-21T10:40:48.687 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:48 smithi175 ceph-mon[25806]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mon metadata", "id": "smithi175"}]: dispatch 2024-09-21T10:40:48.687 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:48 smithi175 ceph-mon[25806]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi123.uqoyow"}]: dispatch 2024-09-21T10:40:48.688 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:48 smithi175 ceph-mon[25806]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi175.jrlevw"}]: dispatch 2024-09-21T10:40:48.688 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:48 smithi175 ceph-mon[25806]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mgr metadata", "who": "smithi123.stnuit", "id": "smithi123.stnuit"}]: dispatch 2024-09-21T10:40:48.688 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:48 smithi175 ceph-mon[25806]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T10:40:48.688 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:48 smithi175 ceph-mon[25806]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T10:40:48.688 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:48 smithi175 ceph-mon[25806]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T10:40:48.688 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:48 smithi175 ceph-mon[25806]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T10:40:48.688 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:48 smithi175 ceph-mon[25806]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T10:40:48.688 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:48 smithi175 ceph-mon[25806]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T10:40:48.688 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:48 smithi175 ceph-mon[25806]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T10:40:48.688 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:48 smithi175 ceph-mon[25806]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T10:40:48.688 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:48 smithi175 ceph-mon[25806]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T10:40:48.688 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:48 smithi175 ceph-mon[25806]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T10:40:48.688 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:48 smithi175 ceph-mon[25806]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T10:40:48.689 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:48 smithi175 ceph-mon[25806]: Manager daemon smithi123.stnuit is now available 2024-09-21T10:40:48.689 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:48 smithi175 ceph-mon[25806]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:40:48.689 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:48 smithi175 ceph-mon[25806]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T10:40:48.689 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:48 smithi175 ceph-mon[25806]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T10:40:48.689 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:48 smithi175 ceph-mon[25806]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.stnuit/mirror_snapshot_schedule"}]: dispatch 2024-09-21T10:40:48.815 INFO:teuthology.orchestra.run.smithi123.stdout: 5 106692 11.48 MB/sec execute 109 sec latency 50.697 ms 2024-09-21T10:40:49.542 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:49 smithi123 ceph-mon[23709]: mgrmap e30: smithi123.stnuit(active, since 1.64689s) 2024-09-21T10:40:49.542 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:49 smithi123 ceph-mon[23709]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T10:40:49.542 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:49 smithi123 ceph-mon[23709]: from='client.24513 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:40:49.542 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:49 smithi123 ceph-mon[23709]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.stnuit/trash_purge_schedule"}]: dispatch 2024-09-21T10:40:49.543 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:49 smithi123 ceph-mon[23709]: pgmap v3: 97 pgs: 97 active+clean; 341 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:40:49.543 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:49 smithi123 ceph-mon[23709]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' 2024-09-21T10:40:49.543 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:49 smithi123 ceph-mon[23709]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' 2024-09-21T10:40:49.759 INFO:teuthology.orchestra.run.smithi123.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T10:40:49.759 INFO:teuthology.orchestra.run.smithi123.stdout:alertmanager.smithi123 smithi123 *:9093,9094 running (76s) 10s ago 10m 19.7M - 0.27.0 11f11916f8cd 5d550bd97de5 2024-09-21T10:40:49.759 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi123 smithi123 running (10m) 10s ago 10m 6405k - 17.2.0 e1d6a67b021e c459f87d5b50 2024-09-21T10:40:49.760 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi175 smithi175 running (9m) 16s ago 9m 6405k - 17.2.0 e1d6a67b021e e35e422d5583 2024-09-21T10:40:49.760 INFO:teuthology.orchestra.run.smithi123.stdout:grafana.smithi123 smithi123 *:3000 running (67s) 10s ago 10m 42.7M - dad864ee21e9 771df81763ba 2024-09-21T10:40:49.760 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi123.uqoyow smithi123 running (4m) 10s ago 4m 39.2M - 17.2.0 e1d6a67b021e f0a21eaac266 2024-09-21T10:40:49.760 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi175.jrlevw smithi175 running (4m) 16s ago 4m 12.3M - 17.2.0 e1d6a67b021e b21b16f4bc50 2024-09-21T10:40:49.760 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi123.stnuit smithi123 *:8443,9283,8765 running (22s) 10s ago 11m 95.1M - 19.3.0-5103-g5c1cdc3c 8f4d013db793 6812cc5cb56b 2024-09-21T10:40:49.760 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi175.txvwrq smithi175 *:8443,9283 running (2m) 16s ago 9m 520M - 19.3.0-5103-g5c1cdc3c 8f4d013db793 b0be7f732c0b 2024-09-21T10:40:49.760 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi123 smithi123 running (11m) 10s ago 11m 52.5M 2048M 17.2.0 e1d6a67b021e 6ad9a8bc53b7 2024-09-21T10:40:49.760 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi175 smithi175 running (9m) 16s ago 9m 44.7M 2048M 17.2.0 e1d6a67b021e 753f564c8cf5 2024-09-21T10:40:49.760 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.0.0.smithi123.ddatpr smithi123 *:2049 running (4m) 10s ago 4m 263M - 4.0 e1d6a67b021e 36f8cb2281eb 2024-09-21T10:40:49.760 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.1.0.smithi175.wpawer smithi175 *:2049 running (3m) 16s ago 3m 87.7M - 4.0 e1d6a67b021e 2200aeed9200 2024-09-21T10:40:49.760 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi123 smithi123 *:9100 running (60s) 10s ago 10m 8098k - 1.7.0 72c9c2088986 ca7d0b29af81 2024-09-21T10:40:49.760 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi175 smithi175 *:9100 running (32s) 16s ago 9m 7558k - 1.7.0 72c9c2088986 46c4f6b7c149 2024-09-21T10:40:49.760 INFO:teuthology.orchestra.run.smithi123.stdout:osd.0 smithi123 running (8m) 10s ago 8m 389M 1936M 17.2.0 e1d6a67b021e a94edf88c456 2024-09-21T10:40:49.760 INFO:teuthology.orchestra.run.smithi123.stdout:osd.1 smithi123 running (8m) 10s ago 8m 303M 1936M 17.2.0 e1d6a67b021e 59aee515b0f8 2024-09-21T10:40:49.761 INFO:teuthology.orchestra.run.smithi123.stdout:osd.2 smithi123 running (7m) 10s ago 7m 248M 1936M 17.2.0 e1d6a67b021e 7db93f8ce5b5 2024-09-21T10:40:49.761 INFO:teuthology.orchestra.run.smithi123.stdout:osd.3 smithi123 running (7m) 10s ago 7m 369M 1936M 17.2.0 e1d6a67b021e c1672dea7b1f 2024-09-21T10:40:49.761 INFO:teuthology.orchestra.run.smithi123.stdout:osd.4 smithi175 running (7m) 16s ago 7m 349M 2704M 17.2.0 e1d6a67b021e 48fe728c259e 2024-09-21T10:40:49.761 INFO:teuthology.orchestra.run.smithi123.stdout:osd.5 smithi175 running (6m) 16s ago 6m 403M 2704M 17.2.0 e1d6a67b021e e270d5a21b14 2024-09-21T10:40:49.761 INFO:teuthology.orchestra.run.smithi123.stdout:osd.6 smithi175 running (6m) 16s ago 6m 333M 2704M 17.2.0 e1d6a67b021e 9bcbf018eb20 2024-09-21T10:40:49.761 INFO:teuthology.orchestra.run.smithi123.stdout:osd.7 smithi175 running (5m) 16s ago 5m 468M 2704M 17.2.0 e1d6a67b021e e9b8f1f90994 2024-09-21T10:40:49.761 INFO:teuthology.orchestra.run.smithi123.stdout:prometheus.smithi123 smithi123 *:9095 running (38s) 10s ago 9m 43.8M - 2.51.0 1d3b7f56885b b898d7fcfff5 2024-09-21T10:40:49.815 INFO:teuthology.orchestra.run.smithi123.stdout: 5 106883 11.43 MB/sec execute 110 sec latency 67.006 ms 2024-09-21T10:40:49.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:49 smithi175 ceph-mon[25806]: mgrmap e30: smithi123.stnuit(active, since 1.64689s) 2024-09-21T10:40:49.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:49 smithi175 ceph-mon[25806]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T10:40:49.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:49 smithi175 ceph-mon[25806]: from='client.24513 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:40:49.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:49 smithi175 ceph-mon[25806]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.stnuit/trash_purge_schedule"}]: dispatch 2024-09-21T10:40:49.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:49 smithi175 ceph-mon[25806]: pgmap v3: 97 pgs: 97 active+clean; 341 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:40:49.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:49 smithi175 ceph-mon[25806]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' 2024-09-21T10:40:49.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:49 smithi175 ceph-mon[25806]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' 2024-09-21T10:40:50.453 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-09-21T10:40:50.453 INFO:teuthology.orchestra.run.smithi123.stdout: "mon": { 2024-09-21T10:40:50.453 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T10:40:50.454 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:40:50.454 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr": { 2024-09-21T10:40:50.454 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev)": 1 2024-09-21T10:40:50.454 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:40:50.454 INFO:teuthology.orchestra.run.smithi123.stdout: "osd": { 2024-09-21T10:40:50.454 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-21T10:40:50.454 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:40:50.454 INFO:teuthology.orchestra.run.smithi123.stdout: "mds": { 2024-09-21T10:40:50.454 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T10:40:50.454 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:40:50.454 INFO:teuthology.orchestra.run.smithi123.stdout: "overall": { 2024-09-21T10:40:50.454 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-09-21T10:40:50.455 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev)": 1 2024-09-21T10:40:50.455 INFO:teuthology.orchestra.run.smithi123.stdout: } 2024-09-21T10:40:50.455 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-09-21T10:40:50.571 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:50 smithi123 ceph-mon[23709]: mgrmap e31: smithi123.stnuit(active, since 2s) 2024-09-21T10:40:50.571 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:50 smithi123 ceph-mon[23709]: [21/Sep/2024:10:40:49] ENGINE Bus STARTING 2024-09-21T10:40:50.815 INFO:teuthology.orchestra.run.smithi123.stdout: 5 107194 11.36 MB/sec execute 111 sec latency 54.635 ms 2024-09-21T10:40:50.848 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:50 smithi175 ceph-mon[25806]: mgrmap e31: smithi123.stnuit(active, since 2s) 2024-09-21T10:40:50.848 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:50 smithi175 ceph-mon[25806]: [21/Sep/2024:10:40:49] ENGINE Bus STARTING 2024-09-21T10:40:50.922 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-09-21T10:40:50.922 INFO:teuthology.orchestra.run.smithi123.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472", 2024-09-21T10:40:50.922 INFO:teuthology.orchestra.run.smithi123.stdout: "in_progress": true, 2024-09-21T10:40:50.922 INFO:teuthology.orchestra.run.smithi123.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T10:40:50.922 INFO:teuthology.orchestra.run.smithi123.stdout: "services_complete": [ 2024-09-21T10:40:50.922 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr" 2024-09-21T10:40:50.923 INFO:teuthology.orchestra.run.smithi123.stdout: ], 2024-09-21T10:40:50.923 INFO:teuthology.orchestra.run.smithi123.stdout: "progress": "2/23 daemons upgraded", 2024-09-21T10:40:50.923 INFO:teuthology.orchestra.run.smithi123.stdout: "message": "", 2024-09-21T10:40:50.923 INFO:teuthology.orchestra.run.smithi123.stdout: "is_paused": false 2024-09-21T10:40:50.923 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-09-21T10:40:51.523 INFO:teuthology.orchestra.run.smithi123.stdout:HEALTH_OK 2024-09-21T10:40:51.604 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:51 smithi123 ceph-mon[23709]: [21/Sep/2024:10:40:50] ENGINE Serving on https://172.21.15.123:7150 2024-09-21T10:40:51.604 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:51 smithi123 ceph-mon[23709]: [21/Sep/2024:10:40:50] ENGINE Error in HTTPServer.serve 2024-09-21T10:40:51.604 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Traceback (most recent call last): 2024-09-21T10:40:51.605 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T10:40:51.605 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: self._connections.run(self.expiration_interval) 2024-09-21T10:40:51.605 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T10:40:51.605 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: self._run(expiration_interval) 2024-09-21T10:40:51.605 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T10:40:51.605 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T10:40:51.605 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T10:40:51.605 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T10:40:51.605 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T10:40:51.605 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: s = self.context.wrap_socket( 2024-09-21T10:40:51.605 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T10:40:51.605 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: return self.sslsocket_class._create( 2024-09-21T10:40:51.605 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T10:40:51.606 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: self.do_handshake() 2024-09-21T10:40:51.606 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T10:40:51.606 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: self._sslobj.do_handshake() 2024-09-21T10:40:51.606 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T10:40:51.606 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:51 smithi123 ceph-mon[23709]: [21/Sep/2024:10:40:50] ENGINE Serving on http://172.21.15.123:8765 2024-09-21T10:40:51.606 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:51 smithi123 ceph-mon[23709]: [21/Sep/2024:10:40:50] ENGINE Bus STARTED 2024-09-21T10:40:51.606 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:51 smithi123 ceph-mon[23709]: from='client.? 172.21.15.123:0/2109315016' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T10:40:51.606 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:51 smithi123 ceph-mon[23709]: from='client.14818 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:40:51.815 INFO:teuthology.orchestra.run.smithi123.stdout: 5 107426 11.31 MB/sec execute 112 sec latency 52.515 ms 2024-09-21T10:40:51.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:51 smithi175 ceph-mon[25806]: [21/Sep/2024:10:40:50] ENGINE Serving on https://172.21.15.123:7150 2024-09-21T10:40:51.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:51 smithi175 ceph-mon[25806]: [21/Sep/2024:10:40:50] ENGINE Error in HTTPServer.serve 2024-09-21T10:40:51.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: Traceback (most recent call last): 2024-09-21T10:40:51.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T10:40:51.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: self._connections.run(self.expiration_interval) 2024-09-21T10:40:51.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T10:40:51.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: self._run(expiration_interval) 2024-09-21T10:40:51.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T10:40:51.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T10:40:51.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T10:40:51.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T10:40:51.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T10:40:51.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: s = self.context.wrap_socket( 2024-09-21T10:40:51.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T10:40:51.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: return self.sslsocket_class._create( 2024-09-21T10:40:51.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T10:40:51.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: self.do_handshake() 2024-09-21T10:40:51.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T10:40:51.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: self._sslobj.do_handshake() 2024-09-21T10:40:51.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T10:40:51.875 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:51 smithi175 ceph-mon[25806]: [21/Sep/2024:10:40:50] ENGINE Serving on http://172.21.15.123:8765 2024-09-21T10:40:51.875 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:51 smithi175 ceph-mon[25806]: [21/Sep/2024:10:40:50] ENGINE Bus STARTED 2024-09-21T10:40:51.875 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:51 smithi175 ceph-mon[25806]: from='client.? 172.21.15.123:0/2109315016' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T10:40:51.875 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:51 smithi175 ceph-mon[25806]: from='client.14818 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:40:52.615 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:52 smithi123 ceph-mon[23709]: pgmap v5: 97 pgs: 97 active+clean; 341 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:40:52.615 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:52 smithi123 ceph-mon[23709]: from='client.? 172.21.15.123:0/3553256904' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T10:40:52.615 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:52 smithi123 ceph-mon[23709]: mgrmap e32: smithi123.stnuit(active, since 4s) 2024-09-21T10:40:52.815 INFO:teuthology.orchestra.run.smithi123.stdout: 5 107711 11.25 MB/sec execute 113 sec latency 108.871 ms 2024-09-21T10:40:52.822 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:52 smithi175 ceph-mon[25806]: pgmap v5: 97 pgs: 97 active+clean; 341 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:40:52.822 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:52 smithi175 ceph-mon[25806]: from='client.? 172.21.15.123:0/3553256904' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T10:40:52.822 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:52 smithi175 ceph-mon[25806]: mgrmap e32: smithi123.stnuit(active, since 4s) 2024-09-21T10:40:53.815 INFO:teuthology.orchestra.run.smithi123.stdout: 5 107953 11.19 MB/sec execute 114 sec latency 246.302 ms 2024-09-21T10:40:54.400 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:54 smithi123 ceph-mon[23709]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' 2024-09-21T10:40:54.400 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:54 smithi123 ceph-mon[23709]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' 2024-09-21T10:40:54.400 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:54 smithi123 ceph-mon[23709]: pgmap v6: 97 pgs: 97 active+clean; 341 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:40:54.400 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:54 smithi123 ceph-mon[23709]: from='mgr.? 172.21.15.175:0/1151229255' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi175.txvwrq/crt"}]: dispatch 2024-09-21T10:40:54.400 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:54 smithi123 ceph-mon[23709]: from='mgr.? 172.21.15.175:0/1151229255' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T10:40:54.400 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:54 smithi123 ceph-mon[23709]: from='mgr.? 172.21.15.175:0/1151229255' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi175.txvwrq/key"}]: dispatch 2024-09-21T10:40:54.400 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:54 smithi123 ceph-mon[23709]: Standby manager daemon smithi175.txvwrq started 2024-09-21T10:40:54.401 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:54 smithi123 ceph-mon[23709]: from='mgr.? 172.21.15.175:0/1151229255' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T10:40:54.401 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:54 smithi123 ceph-mon[23709]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' 2024-09-21T10:40:54.401 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:54 smithi123 ceph-mon[23709]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' 2024-09-21T10:40:54.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:54 smithi175 ceph-mon[25806]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' 2024-09-21T10:40:54.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:54 smithi175 ceph-mon[25806]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' 2024-09-21T10:40:54.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:54 smithi175 ceph-mon[25806]: pgmap v6: 97 pgs: 97 active+clean; 341 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:40:54.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:54 smithi175 ceph-mon[25806]: from='mgr.? 172.21.15.175:0/1151229255' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi175.txvwrq/crt"}]: dispatch 2024-09-21T10:40:54.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:54 smithi175 ceph-mon[25806]: from='mgr.? 172.21.15.175:0/1151229255' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T10:40:54.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:54 smithi175 ceph-mon[25806]: from='mgr.? 172.21.15.175:0/1151229255' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi175.txvwrq/key"}]: dispatch 2024-09-21T10:40:54.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:54 smithi175 ceph-mon[25806]: Standby manager daemon smithi175.txvwrq started 2024-09-21T10:40:54.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:54 smithi175 ceph-mon[25806]: from='mgr.? 172.21.15.175:0/1151229255' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T10:40:54.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:54 smithi175 ceph-mon[25806]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' 2024-09-21T10:40:54.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:54 smithi175 ceph-mon[25806]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' 2024-09-21T10:40:54.815 INFO:teuthology.orchestra.run.smithi123.stdout: 5 108292 11.19 MB/sec execute 115 sec latency 54.465 ms 2024-09-21T10:40:55.419 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:55 smithi123 ceph-mon[23709]: mgrmap e33: smithi123.stnuit(active, since 7s), standbys: smithi175.txvwrq 2024-09-21T10:40:55.419 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:55 smithi123 ceph-mon[23709]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mgr metadata", "who": "smithi175.txvwrq", "id": "smithi175.txvwrq"}]: dispatch 2024-09-21T10:40:55.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:55 smithi175 ceph-mon[25806]: mgrmap e33: smithi123.stnuit(active, since 7s), standbys: smithi175.txvwrq 2024-09-21T10:40:55.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:55 smithi175 ceph-mon[25806]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mgr metadata", "who": "smithi175.txvwrq", "id": "smithi175.txvwrq"}]: dispatch 2024-09-21T10:40:55.815 INFO:teuthology.orchestra.run.smithi123.stdout: 5 108632 11.14 MB/sec execute 116 sec latency 73.831 ms 2024-09-21T10:40:56.815 INFO:teuthology.orchestra.run.smithi123.stdout: 5 109008 11.12 MB/sec execute 117 sec latency 143.847 ms 2024-09-21T10:40:56.856 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:56 smithi123 ceph-mon[23709]: pgmap v7: 97 pgs: 97 active+clean; 330 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 33 KiB/s rd, 2.7 MiB/s wr, 318 op/s 2024-09-21T10:40:57.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:56 smithi175 ceph-mon[25806]: pgmap v7: 97 pgs: 97 active+clean; 330 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 33 KiB/s rd, 2.7 MiB/s wr, 318 op/s 2024-09-21T10:40:57.815 INFO:teuthology.orchestra.run.smithi123.stdout: 5 109298 11.11 MB/sec execute 118 sec latency 155.131 ms 2024-09-21T10:40:58.241 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:57 smithi123 ceph-mon[23709]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' 2024-09-21T10:40:58.241 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:57 smithi123 ceph-mon[23709]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' 2024-09-21T10:40:58.241 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:57 smithi123 ceph-mon[23709]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' 2024-09-21T10:40:58.241 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:57 smithi123 ceph-mon[23709]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' 2024-09-21T10:40:58.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:57 smithi175 ceph-mon[25806]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' 2024-09-21T10:40:58.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:57 smithi175 ceph-mon[25806]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' 2024-09-21T10:40:58.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:57 smithi175 ceph-mon[25806]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' 2024-09-21T10:40:58.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:57 smithi175 ceph-mon[25806]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' 2024-09-21T10:40:58.815 INFO:teuthology.orchestra.run.smithi123.stdout: 5 109742 11.19 MB/sec execute 119 sec latency 63.397 ms 2024-09-21T10:40:59.248 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:40:58 smithi123 ceph-mon[23709]: pgmap v8: 97 pgs: 97 active+clean; 330 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 2.1 MiB/s wr, 246 op/s 2024-09-21T10:40:59.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:40:58 smithi175 ceph-mon[25806]: pgmap v8: 97 pgs: 97 active+clean; 330 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 2.1 MiB/s wr, 246 op/s 2024-09-21T10:40:59.815 INFO:teuthology.orchestra.run.smithi123.stdout: 5 110233 11.24 MB/sec execute 120 sec latency 28.455 ms 2024-09-21T10:41:00.815 INFO:teuthology.orchestra.run.smithi123.stdout: 5 110459 11.19 MB/sec execute 121 sec latency 56.396 ms 2024-09-21T10:41:01.059 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:01 smithi123 ceph-mon[23709]: pgmap v9: 97 pgs: 97 active+clean; 332 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 3.9 MiB/s wr, 417 op/s 2024-09-21T10:41:01.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:01 smithi175 ceph-mon[25806]: pgmap v9: 97 pgs: 97 active+clean; 332 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 3.9 MiB/s wr, 417 op/s 2024-09-21T10:41:01.815 INFO:teuthology.orchestra.run.smithi123.stdout: 5 110680 11.12 MB/sec execute 122 sec latency 94.487 ms 2024-09-21T10:41:02.815 INFO:teuthology.orchestra.run.smithi123.stdout: 5 110994 11.07 MB/sec execute 123 sec latency 49.515 ms 2024-09-21T10:41:03.085 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:03 smithi123 ceph-mon[23709]: pgmap v10: 97 pgs: 97 active+clean; 332 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 3.6 MiB/s wr, 378 op/s 2024-09-21T10:41:03.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:03 smithi175 ceph-mon[25806]: pgmap v10: 97 pgs: 97 active+clean; 332 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 3.6 MiB/s wr, 378 op/s 2024-09-21T10:41:03.815 INFO:teuthology.orchestra.run.smithi123.stdout: 5 111343 11.03 MB/sec execute 124 sec latency 56.224 ms 2024-09-21T10:41:04.106 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:04 smithi123 ceph-mon[23709]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T10:41:04.106 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:04 smithi123 ceph-mon[23709]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' 2024-09-21T10:41:04.106 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:04 smithi123 ceph-mon[23709]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' 2024-09-21T10:41:04.106 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:04 smithi123 ceph-mon[23709]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:41:04.106 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:04 smithi123 ceph-mon[23709]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T10:41:04.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:04 smithi175 ceph-mon[25806]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T10:41:04.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:04 smithi175 ceph-mon[25806]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' 2024-09-21T10:41:04.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:04 smithi175 ceph-mon[25806]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' 2024-09-21T10:41:04.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:04 smithi175 ceph-mon[25806]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:41:04.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:04 smithi175 ceph-mon[25806]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T10:41:04.815 INFO:teuthology.orchestra.run.smithi123.stdout: 5 111636 10.98 MB/sec execute 125 sec latency 73.760 ms 2024-09-21T10:41:05.136 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:05 smithi123 ceph-mon[23709]: pgmap v11: 97 pgs: 97 active+clean; 332 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 3.6 MiB/s wr, 378 op/s 2024-09-21T10:41:05.136 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:05 smithi123 ceph-mon[23709]: Updating smithi123:/etc/ceph/ceph.conf 2024-09-21T10:41:05.137 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:05 smithi123 ceph-mon[23709]: Updating smithi175:/etc/ceph/ceph.conf 2024-09-21T10:41:05.137 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:05 smithi123 ceph-mon[23709]: Updating smithi123:/var/lib/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T10:41:05.137 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:05 smithi123 ceph-mon[23709]: Updating smithi175:/var/lib/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T10:41:05.137 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:05 smithi123 ceph-mon[23709]: Updating smithi123:/etc/ceph/ceph.client.admin.keyring 2024-09-21T10:41:05.137 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:05 smithi123 ceph-mon[23709]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' 2024-09-21T10:41:05.137 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:05 smithi123 ceph-mon[23709]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' 2024-09-21T10:41:05.137 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:05 smithi123 ceph-mon[23709]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' 2024-09-21T10:41:05.137 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:05 smithi123 ceph-mon[23709]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' 2024-09-21T10:41:05.137 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:05 smithi123 ceph-mon[23709]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' 2024-09-21T10:41:05.137 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:05 smithi123 ceph-mon[23709]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' 2024-09-21T10:41:05.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:05 smithi175 ceph-mon[25806]: pgmap v11: 97 pgs: 97 active+clean; 332 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 3.6 MiB/s wr, 378 op/s 2024-09-21T10:41:05.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:05 smithi175 ceph-mon[25806]: Updating smithi123:/etc/ceph/ceph.conf 2024-09-21T10:41:05.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:05 smithi175 ceph-mon[25806]: Updating smithi175:/etc/ceph/ceph.conf 2024-09-21T10:41:05.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:05 smithi175 ceph-mon[25806]: Updating smithi123:/var/lib/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T10:41:05.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:05 smithi175 ceph-mon[25806]: Updating smithi175:/var/lib/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T10:41:05.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:05 smithi175 ceph-mon[25806]: Updating smithi123:/etc/ceph/ceph.client.admin.keyring 2024-09-21T10:41:05.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:05 smithi175 ceph-mon[25806]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' 2024-09-21T10:41:05.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:05 smithi175 ceph-mon[25806]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' 2024-09-21T10:41:05.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:05 smithi175 ceph-mon[25806]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' 2024-09-21T10:41:05.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:05 smithi175 ceph-mon[25806]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' 2024-09-21T10:41:05.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:05 smithi175 ceph-mon[25806]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' 2024-09-21T10:41:05.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:05 smithi175 ceph-mon[25806]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' 2024-09-21T10:41:05.815 INFO:teuthology.orchestra.run.smithi123.stdout: 5 112009 11.01 MB/sec execute 126 sec latency 97.215 ms 2024-09-21T10:41:06.160 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:06 smithi123 ceph-mon[23709]: Updating smithi175:/etc/ceph/ceph.client.admin.keyring 2024-09-21T10:41:06.160 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:06 smithi123 ceph-mon[23709]: Updating smithi123:/var/lib/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T10:41:06.160 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:06 smithi123 ceph-mon[23709]: Updating smithi175:/var/lib/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T10:41:06.161 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:06 smithi123 ceph-mon[23709]: Reconfiguring prometheus.smithi123 (dependencies changed)... 2024-09-21T10:41:06.161 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:06 smithi123 ceph-mon[23709]: Reconfiguring daemon prometheus.smithi123 on smithi123 2024-09-21T10:41:06.161 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:06 smithi123 ceph-mon[23709]: pgmap v12: 97 pgs: 97 active+clean; 336 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 5.6 MiB/s wr, 583 op/s 2024-09-21T10:41:06.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:06 smithi175 ceph-mon[25806]: Updating smithi175:/etc/ceph/ceph.client.admin.keyring 2024-09-21T10:41:06.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:06 smithi175 ceph-mon[25806]: Updating smithi123:/var/lib/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T10:41:06.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:06 smithi175 ceph-mon[25806]: Updating smithi175:/var/lib/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T10:41:06.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:06 smithi175 ceph-mon[25806]: Reconfiguring prometheus.smithi123 (dependencies changed)... 2024-09-21T10:41:06.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:06 smithi175 ceph-mon[25806]: Reconfiguring daemon prometheus.smithi123 on smithi123 2024-09-21T10:41:06.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:06 smithi175 ceph-mon[25806]: pgmap v12: 97 pgs: 97 active+clean; 336 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 5.6 MiB/s wr, 583 op/s 2024-09-21T10:41:06.815 INFO:teuthology.orchestra.run.smithi123.stdout: 5 112352 10.95 MB/sec execute 127 sec latency 55.998 ms 2024-09-21T10:41:07.815 INFO:teuthology.orchestra.run.smithi123.stdout: 5 112729 10.97 MB/sec execute 128 sec latency 97.644 ms 2024-09-21T10:41:08.711 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:08 smithi123 ceph-mon[23709]: pgmap v13: 97 pgs: 97 active+clean; 336 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.4 KiB/s rd, 4.1 MiB/s wr, 400 op/s 2024-09-21T10:41:08.815 INFO:teuthology.orchestra.run.smithi123.stdout: 5 112991 10.96 MB/sec execute 129 sec latency 64.230 ms 2024-09-21T10:41:08.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:08 smithi175 ceph-mon[25806]: pgmap v13: 97 pgs: 97 active+clean; 336 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.4 KiB/s rd, 4.1 MiB/s wr, 400 op/s 2024-09-21T10:41:09.815 INFO:teuthology.orchestra.run.smithi123.stdout: 5 113388 10.99 MB/sec execute 130 sec latency 55.596 ms 2024-09-21T10:41:10.760 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:10 smithi123 ceph-mon[23709]: pgmap v14: 97 pgs: 97 active+clean; 331 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 6.0 MiB/s wr, 604 op/s 2024-09-21T10:41:10.815 INFO:teuthology.orchestra.run.smithi123.stdout: 5 113733 11.00 MB/sec execute 131 sec latency 53.941 ms 2024-09-21T10:41:10.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:10 smithi175 ceph-mon[25806]: pgmap v14: 97 pgs: 97 active+clean; 331 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 6.0 MiB/s wr, 604 op/s 2024-09-21T10:41:11.815 INFO:teuthology.orchestra.run.smithi123.stdout: 5 113973 10.98 MB/sec execute 132 sec latency 72.694 ms 2024-09-21T10:41:12.807 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:12 smithi123 ceph-mon[23709]: pgmap v15: 97 pgs: 97 active+clean; 331 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.0 MiB/s wr, 408 op/s 2024-09-21T10:41:12.815 INFO:teuthology.orchestra.run.smithi123.stdout: 5 114208 10.92 MB/sec execute 133 sec latency 129.598 ms 2024-09-21T10:41:13.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:12 smithi175 ceph-mon[25806]: pgmap v15: 97 pgs: 97 active+clean; 331 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.0 MiB/s wr, 408 op/s 2024-09-21T10:41:13.815 INFO:teuthology.orchestra.run.smithi123.stdout: 5 114576 10.88 MB/sec execute 134 sec latency 115.104 ms 2024-09-21T10:41:13.957 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:13 smithi175 ceph-mon[25806]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' 2024-09-21T10:41:13.957 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:13 smithi175 ceph-mon[25806]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' 2024-09-21T10:41:13.957 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:13 smithi175 ceph-mon[25806]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T10:41:13.957 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:13 smithi175 ceph-mon[25806]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T10:41:13.957 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:13 smithi175 ceph-mon[25806]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:41:14.075 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:13 smithi123 ceph-mon[23709]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' 2024-09-21T10:41:14.075 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:13 smithi123 ceph-mon[23709]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' 2024-09-21T10:41:14.075 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:13 smithi123 ceph-mon[23709]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T10:41:14.075 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:13 smithi123 ceph-mon[23709]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T10:41:14.075 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:13 smithi123 ceph-mon[23709]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:41:14.816 INFO:teuthology.orchestra.run.smithi123.stdout: 5 114899 10.85 MB/sec execute 135 sec latency 120.909 ms 2024-09-21T10:41:15.139 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:14 smithi123 ceph-mon[23709]: pgmap v16: 97 pgs: 97 active+clean; 331 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.0 MiB/s wr, 408 op/s 2024-09-21T10:41:15.366 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:14 smithi175 ceph-mon[25806]: pgmap v16: 97 pgs: 97 active+clean; 331 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.0 MiB/s wr, 408 op/s 2024-09-21T10:41:15.815 INFO:teuthology.orchestra.run.smithi123.stdout: 5 115113 10.79 MB/sec execute 136 sec latency 156.575 ms 2024-09-21T10:41:15.907 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:15 smithi123 ceph-mon[23709]: Upgrade: Updating mgr.smithi175.txvwrq 2024-09-21T10:41:15.907 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:15 smithi123 ceph-mon[23709]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi175.txvwrq", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T10:41:15.907 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:15 smithi123 ceph-mon[23709]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T10:41:15.907 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:15 smithi123 ceph-mon[23709]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:41:15.907 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:15 smithi123 ceph-mon[23709]: Deploying daemon mgr.smithi175.txvwrq on smithi175 2024-09-21T10:41:16.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:15 smithi175 ceph-mon[25806]: Upgrade: Updating mgr.smithi175.txvwrq 2024-09-21T10:41:16.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:15 smithi175 ceph-mon[25806]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi175.txvwrq", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T10:41:16.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:15 smithi175 ceph-mon[25806]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T10:41:16.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:15 smithi175 ceph-mon[25806]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:41:16.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:15 smithi175 ceph-mon[25806]: Deploying daemon mgr.smithi175.txvwrq on smithi175 2024-09-21T10:41:16.815 INFO:teuthology.orchestra.run.smithi123.stdout: 5 115538 10.82 MB/sec execute 137 sec latency 52.486 ms 2024-09-21T10:41:16.918 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:16 smithi123 ceph-mon[23709]: pgmap v17: 97 pgs: 97 active+clean; 342 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 9.1 KiB/s rd, 6.2 MiB/s wr, 593 op/s 2024-09-21T10:41:17.059 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:16 smithi175 ceph-mon[25806]: pgmap v17: 97 pgs: 97 active+clean; 342 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 9.1 KiB/s rd, 6.2 MiB/s wr, 593 op/s 2024-09-21T10:41:17.816 INFO:teuthology.orchestra.run.smithi123.stdout: 5 115915 10.77 MB/sec execute 138 sec latency 57.462 ms 2024-09-21T10:41:18.816 INFO:teuthology.orchestra.run.smithi123.stdout: 5 116198 10.76 MB/sec execute 139 sec latency 88.635 ms 2024-09-21T10:41:19.031 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:18 smithi123 ceph-mon[23709]: pgmap v18: 97 pgs: 97 active+clean; 342 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.1 MiB/s wr, 388 op/s 2024-09-21T10:41:19.031 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:18 smithi123 ceph-mon[23709]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T10:41:19.031 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:18 smithi123 ceph-mon[23709]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' 2024-09-21T10:41:19.031 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:18 smithi123 ceph-mon[23709]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' 2024-09-21T10:41:19.032 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:18 smithi123 ceph-mon[23709]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:41:19.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:18 smithi175 ceph-mon[25806]: pgmap v18: 97 pgs: 97 active+clean; 342 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.1 MiB/s wr, 388 op/s 2024-09-21T10:41:19.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:18 smithi175 ceph-mon[25806]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T10:41:19.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:18 smithi175 ceph-mon[25806]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' 2024-09-21T10:41:19.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:18 smithi175 ceph-mon[25806]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' 2024-09-21T10:41:19.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:18 smithi175 ceph-mon[25806]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:41:19.816 INFO:teuthology.orchestra.run.smithi123.stdout: 5 116474 10.75 MB/sec execute 140 sec latency 69.757 ms 2024-09-21T10:41:20.816 INFO:teuthology.orchestra.run.smithi123.stdout: 5 116854 10.81 MB/sec execute 141 sec latency 47.571 ms 2024-09-21T10:41:21.092 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:20 smithi123 ceph-mon[23709]: pgmap v19: 97 pgs: 97 active+clean; 331 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.9 KiB/s rd, 5.9 MiB/s wr, 591 op/s 2024-09-21T10:41:21.325 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:20 smithi175 ceph-mon[25806]: pgmap v19: 97 pgs: 97 active+clean; 331 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.9 KiB/s rd, 5.9 MiB/s wr, 591 op/s 2024-09-21T10:41:21.816 INFO:teuthology.orchestra.run.smithi123.stdout: 5 117154 10.79 MB/sec execute 142 sec latency 139.451 ms 2024-09-21T10:41:21.960 INFO:teuthology.orchestra.run.smithi123.stdout:true 2024-09-21T10:41:22.816 INFO:teuthology.orchestra.run.smithi123.stdout: 5 117519 10.79 MB/sec execute 143 sec latency 129.414 ms 2024-09-21T10:41:22.966 INFO:teuthology.orchestra.run.smithi123.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T10:41:22.967 INFO:teuthology.orchestra.run.smithi123.stdout:alertmanager.smithi123 smithi123 *:9093,9094 running (109s) 25s ago 11m 19.7M - 0.27.0 11f11916f8cd 5d550bd97de5 2024-09-21T10:41:22.967 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi123 smithi123 running (10m) 25s ago 10m 6405k - 17.2.0 e1d6a67b021e c459f87d5b50 2024-09-21T10:41:22.967 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi175 smithi175 running (10m) 0s ago 10m 6405k - 17.2.0 e1d6a67b021e e35e422d5583 2024-09-21T10:41:22.967 INFO:teuthology.orchestra.run.smithi123.stdout:grafana.smithi123 smithi123 *:3000 running (101s) 25s ago 10m 45.1M - dad864ee21e9 771df81763ba 2024-09-21T10:41:22.967 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi123.uqoyow smithi123 running (4m) 25s ago 4m 39.6M - 17.2.0 e1d6a67b021e f0a21eaac266 2024-09-21T10:41:22.967 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi175.jrlevw smithi175 running (4m) 0s ago 4m 12.7M - 17.2.0 e1d6a67b021e b21b16f4bc50 2024-09-21T10:41:22.967 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi123.stnuit smithi123 *:8443,9283,8765 running (55s) 25s ago 12m 493M - 19.3.0-5103-g5c1cdc3c 8f4d013db793 6812cc5cb56b 2024-09-21T10:41:22.967 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi175.txvwrq smithi175 *:8443,9283,8765 running (4s) 0s ago 10m 68.4M - 19.3.0-5103-g5c1cdc3c 8f4d013db793 304f0408075e 2024-09-21T10:41:22.967 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi123 smithi123 running (12m) 25s ago 12m 55.0M 2048M 17.2.0 e1d6a67b021e 6ad9a8bc53b7 2024-09-21T10:41:22.967 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi175 smithi175 running (10m) 0s ago 10m 45.3M 2048M 17.2.0 e1d6a67b021e 753f564c8cf5 2024-09-21T10:41:22.967 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.0.0.smithi123.ddatpr smithi123 *:2049 running (4m) 25s ago 4m 268M - 4.0 e1d6a67b021e 36f8cb2281eb 2024-09-21T10:41:22.967 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.1.0.smithi175.wpawer smithi175 *:2049 running (4m) 0s ago 4m 88.0M - 4.0 e1d6a67b021e 2200aeed9200 2024-09-21T10:41:22.967 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi123 smithi123 *:9100 running (93s) 25s ago 10m 8238k - 1.7.0 72c9c2088986 ca7d0b29af81 2024-09-21T10:41:22.967 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi175 smithi175 *:9100 running (65s) 0s ago 10m 9.92M - 1.7.0 72c9c2088986 46c4f6b7c149 2024-09-21T10:41:22.967 INFO:teuthology.orchestra.run.smithi123.stdout:osd.0 smithi123 running (9m) 25s ago 9m 412M 1936M 17.2.0 e1d6a67b021e a94edf88c456 2024-09-21T10:41:22.967 INFO:teuthology.orchestra.run.smithi123.stdout:osd.1 smithi123 running (8m) 25s ago 8m 330M 1936M 17.2.0 e1d6a67b021e 59aee515b0f8 2024-09-21T10:41:22.968 INFO:teuthology.orchestra.run.smithi123.stdout:osd.2 smithi123 running (8m) 25s ago 8m 274M 1936M 17.2.0 e1d6a67b021e 7db93f8ce5b5 2024-09-21T10:41:22.968 INFO:teuthology.orchestra.run.smithi123.stdout:osd.3 smithi123 running (8m) 25s ago 8m 395M 1936M 17.2.0 e1d6a67b021e c1672dea7b1f 2024-09-21T10:41:22.968 INFO:teuthology.orchestra.run.smithi123.stdout:osd.4 smithi175 running (7m) 0s ago 7m 390M 2704M 17.2.0 e1d6a67b021e 48fe728c259e 2024-09-21T10:41:22.968 INFO:teuthology.orchestra.run.smithi123.stdout:osd.5 smithi175 running (7m) 0s ago 7m 457M 2704M 17.2.0 e1d6a67b021e e270d5a21b14 2024-09-21T10:41:22.968 INFO:teuthology.orchestra.run.smithi123.stdout:osd.6 smithi175 running (6m) 0s ago 6m 397M 2704M 17.2.0 e1d6a67b021e 9bcbf018eb20 2024-09-21T10:41:22.968 INFO:teuthology.orchestra.run.smithi123.stdout:osd.7 smithi175 running (6m) 0s ago 6m 519M 2704M 17.2.0 e1d6a67b021e e9b8f1f90994 2024-09-21T10:41:22.968 INFO:teuthology.orchestra.run.smithi123.stdout:prometheus.smithi123 smithi123 *:9095 starting - - - - 2024-09-21T10:41:23.116 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:22 smithi123 ceph-mon[23709]: pgmap v20: 97 pgs: 97 active+clean; 331 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.9 MiB/s wr, 387 op/s 2024-09-21T10:41:23.116 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:22 smithi123 ceph-mon[23709]: from='client.14830 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:41:23.116 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:22 smithi123 ceph-mon[23709]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' 2024-09-21T10:41:23.116 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:22 smithi123 ceph-mon[23709]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' 2024-09-21T10:41:23.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:22 smithi175 ceph-mon[25806]: pgmap v20: 97 pgs: 97 active+clean; 331 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.9 MiB/s wr, 387 op/s 2024-09-21T10:41:23.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:22 smithi175 ceph-mon[25806]: from='client.14830 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:41:23.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:22 smithi175 ceph-mon[25806]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' 2024-09-21T10:41:23.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:22 smithi175 ceph-mon[25806]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' 2024-09-21T10:41:23.492 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-09-21T10:41:23.492 INFO:teuthology.orchestra.run.smithi123.stdout: "mon": { 2024-09-21T10:41:23.492 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T10:41:23.492 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:41:23.492 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr": { 2024-09-21T10:41:23.492 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev)": 2 2024-09-21T10:41:23.492 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:41:23.493 INFO:teuthology.orchestra.run.smithi123.stdout: "osd": { 2024-09-21T10:41:23.493 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-21T10:41:23.493 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:41:23.493 INFO:teuthology.orchestra.run.smithi123.stdout: "mds": { 2024-09-21T10:41:23.493 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T10:41:23.493 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:41:23.493 INFO:teuthology.orchestra.run.smithi123.stdout: "overall": { 2024-09-21T10:41:23.493 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-09-21T10:41:23.493 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev)": 2 2024-09-21T10:41:23.493 INFO:teuthology.orchestra.run.smithi123.stdout: } 2024-09-21T10:41:23.493 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-09-21T10:41:23.816 INFO:teuthology.orchestra.run.smithi123.stdout: 5 117752 10.76 MB/sec execute 144 sec latency 46.890 ms 2024-09-21T10:41:23.974 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-09-21T10:41:23.974 INFO:teuthology.orchestra.run.smithi123.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472", 2024-09-21T10:41:23.974 INFO:teuthology.orchestra.run.smithi123.stdout: "in_progress": true, 2024-09-21T10:41:23.974 INFO:teuthology.orchestra.run.smithi123.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T10:41:23.974 INFO:teuthology.orchestra.run.smithi123.stdout: "services_complete": [ 2024-09-21T10:41:23.974 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr" 2024-09-21T10:41:23.974 INFO:teuthology.orchestra.run.smithi123.stdout: ], 2024-09-21T10:41:23.974 INFO:teuthology.orchestra.run.smithi123.stdout: "progress": "2/23 daemons upgraded", 2024-09-21T10:41:23.974 INFO:teuthology.orchestra.run.smithi123.stdout: "message": "Currently upgrading mgr daemons", 2024-09-21T10:41:23.974 INFO:teuthology.orchestra.run.smithi123.stdout: "is_paused": false 2024-09-21T10:41:23.975 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-09-21T10:41:24.137 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:23 smithi123 ceph-mon[23709]: from='client.14834 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:41:24.137 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:23 smithi123 ceph-mon[23709]: from='client.14838 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:41:24.137 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:23 smithi123 ceph-mon[23709]: from='client.? 172.21.15.123:0/1077258689' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T10:41:24.137 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:23 smithi123 ceph-mon[23709]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' 2024-09-21T10:41:24.137 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:23 smithi123 ceph-mon[23709]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' 2024-09-21T10:41:24.297 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:23 smithi175 ceph-mon[25806]: from='client.14834 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:41:24.297 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:23 smithi175 ceph-mon[25806]: from='client.14838 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:41:24.297 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:23 smithi175 ceph-mon[25806]: from='client.? 172.21.15.123:0/1077258689' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T10:41:24.297 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:23 smithi175 ceph-mon[25806]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' 2024-09-21T10:41:24.297 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:23 smithi175 ceph-mon[25806]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' 2024-09-21T10:41:24.529 INFO:teuthology.orchestra.run.smithi123.stdout:HEALTH_OK 2024-09-21T10:41:24.816 INFO:teuthology.orchestra.run.smithi123.stdout: 5 118002 10.70 MB/sec execute 145 sec latency 80.558 ms 2024-09-21T10:41:25.168 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:25 smithi123 ceph-mon[23709]: pgmap v21: 97 pgs: 97 active+clean; 331 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.9 MiB/s wr, 387 op/s 2024-09-21T10:41:25.168 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:25 smithi123 ceph-mon[23709]: from='client.24557 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:41:25.168 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:25 smithi123 ceph-mon[23709]: from='client.? 172.21.15.123:0/2983906477' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T10:41:25.168 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:25 smithi123 ceph-mon[23709]: from='mgr.? 172.21.15.175:0/3786124703' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi175.txvwrq/crt"}]: dispatch 2024-09-21T10:41:25.168 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:25 smithi123 ceph-mon[23709]: from='mgr.? 172.21.15.175:0/3786124703' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T10:41:25.168 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:25 smithi123 ceph-mon[23709]: from='mgr.? 172.21.15.175:0/3786124703' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi175.txvwrq/key"}]: dispatch 2024-09-21T10:41:25.168 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:25 smithi123 ceph-mon[23709]: from='mgr.? 172.21.15.175:0/3786124703' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T10:41:25.168 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:25 smithi123 ceph-mon[23709]: Standby manager daemon smithi175.txvwrq restarted 2024-09-21T10:41:25.168 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:25 smithi123 ceph-mon[23709]: Standby manager daemon smithi175.txvwrq started 2024-09-21T10:41:25.248 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:25 smithi175 ceph-mon[25806]: pgmap v21: 97 pgs: 97 active+clean; 331 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.9 MiB/s wr, 387 op/s 2024-09-21T10:41:25.248 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:25 smithi175 ceph-mon[25806]: from='client.24557 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:41:25.249 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:25 smithi175 ceph-mon[25806]: from='client.? 172.21.15.123:0/2983906477' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T10:41:25.249 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:25 smithi175 ceph-mon[25806]: from='mgr.? 172.21.15.175:0/3786124703' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi175.txvwrq/crt"}]: dispatch 2024-09-21T10:41:25.249 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:25 smithi175 ceph-mon[25806]: from='mgr.? 172.21.15.175:0/3786124703' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T10:41:25.249 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:25 smithi175 ceph-mon[25806]: from='mgr.? 172.21.15.175:0/3786124703' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi175.txvwrq/key"}]: dispatch 2024-09-21T10:41:25.249 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:25 smithi175 ceph-mon[25806]: from='mgr.? 172.21.15.175:0/3786124703' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T10:41:25.249 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:25 smithi175 ceph-mon[25806]: Standby manager daemon smithi175.txvwrq restarted 2024-09-21T10:41:25.249 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:25 smithi175 ceph-mon[25806]: Standby manager daemon smithi175.txvwrq started 2024-09-21T10:41:25.816 INFO:teuthology.orchestra.run.smithi123.stdout: 5 118366 10.68 MB/sec execute 146 sec latency 56.094 ms 2024-09-21T10:41:26.182 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:26 smithi123 ceph-mon[23709]: mgrmap e34: smithi123.stnuit(active, since 38s), standbys: smithi175.txvwrq 2024-09-21T10:41:26.182 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:26 smithi123 ceph-mon[23709]: pgmap v22: 97 pgs: 97 active+clean; 339 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 6.2 MiB/s wr, 575 op/s 2024-09-21T10:41:26.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:26 smithi175 ceph-mon[25806]: mgrmap e34: smithi123.stnuit(active, since 38s), standbys: smithi175.txvwrq 2024-09-21T10:41:26.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:26 smithi175 ceph-mon[25806]: pgmap v22: 97 pgs: 97 active+clean; 339 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 6.2 MiB/s wr, 575 op/s 2024-09-21T10:41:26.816 INFO:teuthology.orchestra.run.smithi123.stdout: 5 118651 10.65 MB/sec execute 147 sec latency 82.776 ms 2024-09-21T10:41:27.816 INFO:teuthology.orchestra.run.smithi123.stdout: 5 119088 10.67 MB/sec execute 148 sec latency 115.786 ms 2024-09-21T10:41:27.949 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:27 smithi123 ceph-mon[23709]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' 2024-09-21T10:41:27.949 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:27 smithi123 ceph-mon[23709]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' 2024-09-21T10:41:27.949 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:27 smithi123 ceph-mon[23709]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' 2024-09-21T10:41:27.949 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:27 smithi123 ceph-mon[23709]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' 2024-09-21T10:41:27.949 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:27 smithi123 ceph-mon[23709]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:41:27.949 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:27 smithi123 ceph-mon[23709]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T10:41:27.949 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:27 smithi123 ceph-mon[23709]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' 2024-09-21T10:41:27.950 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:27 smithi123 ceph-mon[23709]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' 2024-09-21T10:41:27.950 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:27 smithi123 ceph-mon[23709]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:41:27.950 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:27 smithi123 ceph-mon[23709]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T10:41:27.950 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:27 smithi123 ceph-mon[23709]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' 2024-09-21T10:41:27.950 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:27 smithi123 ceph-mon[23709]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi123.stnuit"}]: dispatch 2024-09-21T10:41:27.950 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:27 smithi123 ceph-mon[23709]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi123.stnuit"}]': finished 2024-09-21T10:41:27.950 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:27 smithi123 ceph-mon[23709]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi175.txvwrq"}]: dispatch 2024-09-21T10:41:27.950 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:27 smithi123 ceph-mon[23709]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi175.txvwrq"}]': finished 2024-09-21T10:41:27.950 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:27 smithi123 ceph-mon[23709]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-21T10:41:28.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:27 smithi175 ceph-mon[25806]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' 2024-09-21T10:41:28.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:27 smithi175 ceph-mon[25806]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' 2024-09-21T10:41:28.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:27 smithi175 ceph-mon[25806]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' 2024-09-21T10:41:28.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:27 smithi175 ceph-mon[25806]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' 2024-09-21T10:41:28.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:27 smithi175 ceph-mon[25806]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:41:28.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:27 smithi175 ceph-mon[25806]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T10:41:28.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:27 smithi175 ceph-mon[25806]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' 2024-09-21T10:41:28.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:27 smithi175 ceph-mon[25806]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' 2024-09-21T10:41:28.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:27 smithi175 ceph-mon[25806]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:41:28.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:27 smithi175 ceph-mon[25806]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T10:41:28.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:27 smithi175 ceph-mon[25806]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' 2024-09-21T10:41:28.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:27 smithi175 ceph-mon[25806]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi123.stnuit"}]: dispatch 2024-09-21T10:41:28.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:27 smithi175 ceph-mon[25806]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi123.stnuit"}]': finished 2024-09-21T10:41:28.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:27 smithi175 ceph-mon[25806]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi175.txvwrq"}]: dispatch 2024-09-21T10:41:28.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:27 smithi175 ceph-mon[25806]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi175.txvwrq"}]': finished 2024-09-21T10:41:28.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:27 smithi175 ceph-mon[25806]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-21T10:41:28.816 INFO:teuthology.orchestra.run.smithi123.stdout: 5 119481 10.63 MB/sec execute 149 sec latency 92.245 ms 2024-09-21T10:41:28.974 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:28 smithi123 ceph-mon[23709]: Upgrade: Setting container_image for all mgr 2024-09-21T10:41:28.974 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:28 smithi123 ceph-mon[23709]: pgmap v23: 97 pgs: 97 active+clean; 339 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.0 MiB/s wr, 390 op/s 2024-09-21T10:41:29.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:28 smithi175 ceph-mon[25806]: Upgrade: Setting container_image for all mgr 2024-09-21T10:41:29.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:28 smithi175 ceph-mon[25806]: pgmap v23: 97 pgs: 97 active+clean; 339 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.0 MiB/s wr, 390 op/s 2024-09-21T10:41:29.816 INFO:teuthology.orchestra.run.smithi123.stdout: 5 119815 10.63 MB/sec execute 150 sec latency 60.540 ms 2024-09-21T10:41:30.816 INFO:teuthology.orchestra.run.smithi123.stdout: 5 120077 10.63 MB/sec execute 151 sec latency 50.696 ms 2024-09-21T10:41:30.827 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:30 smithi123 ceph-mon[23709]: pgmap v24: 97 pgs: 97 active+clean; 334 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 9.4 KiB/s rd, 5.6 MiB/s wr, 596 op/s 2024-09-21T10:41:30.827 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:30 smithi123 ceph-mon[23709]: Upgrade: Updating mon.smithi123 2024-09-21T10:41:30.827 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:30 smithi123 ceph-mon[23709]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' 2024-09-21T10:41:30.827 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:30 smithi123 ceph-mon[23709]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T10:41:30.827 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:30 smithi123 ceph-mon[23709]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T10:41:30.827 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:30 smithi123 ceph-mon[23709]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:41:30.827 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:30 smithi123 ceph-mon[23709]: Deploying daemon mon.smithi123 on smithi123 2024-09-21T10:41:31.035 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:30 smithi175 ceph-mon[25806]: pgmap v24: 97 pgs: 97 active+clean; 334 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 9.4 KiB/s rd, 5.6 MiB/s wr, 596 op/s 2024-09-21T10:41:31.036 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:30 smithi175 ceph-mon[25806]: Upgrade: Updating mon.smithi123 2024-09-21T10:41:31.036 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:30 smithi175 ceph-mon[25806]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' 2024-09-21T10:41:31.036 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:30 smithi175 ceph-mon[25806]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T10:41:31.036 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:30 smithi175 ceph-mon[25806]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T10:41:31.036 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:30 smithi175 ceph-mon[25806]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:41:31.036 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:30 smithi175 ceph-mon[25806]: Deploying daemon mon.smithi123 on smithi123 2024-09-21T10:41:31.816 INFO:teuthology.orchestra.run.smithi123.stdout: 5 120391 10.64 MB/sec execute 152 sec latency 82.229 ms 2024-09-21T10:41:32.124 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:32 smithi123 systemd[1]: Stopping Ceph mon.smithi123 for 3ccd15e6-7804-11ef-baf6-efdc52797490... 2024-09-21T10:41:32.375 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:32 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-mon-smithi123[23705]: 2024-09-21T10:41:32.360+0000 7fa323a71700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi123 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:41:32.375 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:32 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-mon-smithi123[23705]: 2024-09-21T10:41:32.360+0000 7fa323a71700 -1 mon.smithi123@0(leader) e2 *** Got Signal Terminated *** 2024-09-21T10:41:32.663 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:32 smithi123 podman[63690]: 2024-09-21 10:41:32.404480478 +0000 UTC m=+0.160189592 container died 6ad9a8bc53b7409b926333dae2fdd2583283bec41caf28164c54db139d92f6a1 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-mon-smithi123, ceph=True, io.buildah.version=1.19.8, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, vcs-type=git, GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-09-21T10:41:32.816 INFO:teuthology.orchestra.run.smithi123.stdout: 5 120601 10.62 MB/sec execute 153 sec latency 84.753 ms 2024-09-21T10:41:33.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:33 smithi123 podman[63690]: 2024-09-21 10:41:33.024879508 +0000 UTC m=+0.780588617 container cleanup 6ad9a8bc53b7409b926333dae2fdd2583283bec41caf28164c54db139d92f6a1 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-mon-smithi123, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, io.buildah.version=1.19.8, distribution-scope=public, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, architecture=x86_64, io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, version=8, release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux ) 2024-09-21T10:41:33.169 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:33 smithi123 bash[63690]: ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-mon-smithi123 2024-09-21T10:41:33.671 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:33 smithi123 podman[63702]: 2024-09-21 10:41:33.632236964 +0000 UTC m=+1.231826721 container remove 6ad9a8bc53b7409b926333dae2fdd2583283bec41caf28164c54db139d92f6a1 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-mon-smithi123, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, vcs-type=git, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., RELEASE=HEAD, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-09-21T10:41:33.816 INFO:teuthology.orchestra.run.smithi123.stdout: 5 120949 10.63 MB/sec execute 154 sec latency 51.218 ms 2024-09-21T10:41:34.176 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:33 smithi123 systemd[1]: ceph-3ccd15e6-7804-11ef-baf6-efdc52797490@mon.smithi123.service: Deactivated successfully. 2024-09-21T10:41:34.176 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:33 smithi123 systemd[1]: Stopped Ceph mon.smithi123 for 3ccd15e6-7804-11ef-baf6-efdc52797490. 2024-09-21T10:41:34.176 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:33 smithi123 systemd[1]: ceph-3ccd15e6-7804-11ef-baf6-efdc52797490@mon.smithi123.service: Consumed 15.489s CPU time. 2024-09-21T10:41:34.435 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 systemd[1]: Starting Ceph mon.smithi123 for 3ccd15e6-7804-11ef-baf6-efdc52797490... 2024-09-21T10:41:34.703 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 podman[63838]: 2024-09-21 10:41:34.365535784 +0000 UTC m=+0.018337721 image pull 8f4d013db793795fef09ff5edf96e6b42ba8cd2c9158b2f0b7a5d7c1e657c063 quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472 2024-09-21T10:41:34.703 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 podman[63838]: 2024-09-21 10:41:34.509723163 +0000 UTC m=+0.162525098 container create 6b95207a2d807be3959310629d20a58fbce105e32dffa46adcf09109666a4896 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-mon-smithi123, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=) 2024-09-21T10:41:34.816 INFO:teuthology.orchestra.run.smithi123.stdout: 5 121238 10.59 MB/sec execute 155 sec latency 44.526 ms 2024-09-21T10:41:34.962 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 podman[63838]: 2024-09-21 10:41:34.876702728 +0000 UTC m=+0.529504666 container init 6b95207a2d807be3959310629d20a58fbce105e32dffa46adcf09109666a4896 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-mon-smithi123, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, ceph=True, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T10:41:34.962 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 podman[63838]: 2024-09-21 10:41:34.888653124 +0000 UTC m=+0.541455066 container start 6b95207a2d807be3959310629d20a58fbce105e32dffa46adcf09109666a4896 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-mon-smithi123, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-09-21T10:41:34.962 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: set uid:gid to 167:167 (ceph:ceph) 2024-09-21T10:41:34.962 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev), process ceph-mon, pid 2 2024-09-21T10:41:34.962 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: pidfile_write: ignore empty --pid-file 2024-09-21T10:41:34.962 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: load: jerasure load: lrc 2024-09-21T10:41:34.962 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: RocksDB version: 7.9.2 2024-09-21T10:41:34.962 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Git sha 0 2024-09-21T10:41:34.963 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Compile date 2024-09-21 08:36:07 2024-09-21T10:41:34.963 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: DB SUMMARY 2024-09-21T10:41:34.963 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: DB Session ID: ET1QPGT5D6U09IP9RBK3 2024-09-21T10:41:34.963 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: CURRENT file: CURRENT 2024-09-21T10:41:34.963 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: IDENTITY file: IDENTITY 2024-09-21T10:41:34.963 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: MANIFEST file: MANIFEST-000015 size: 978 Bytes 2024-09-21T10:41:34.963 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi123/store.db dir, Total Num: 1, files: 000030.sst 2024-09-21T10:41:34.963 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi123/store.db: 000028.log size: 2473634 ; 2024-09-21T10:41:34.963 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.error_if_exists: 0 2024-09-21T10:41:34.963 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.create_if_missing: 0 2024-09-21T10:41:34.963 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.paranoid_checks: 1 2024-09-21T10:41:34.963 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-21T10:41:34.963 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-21T10:41:34.964 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-21T10:41:34.964 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.env: 0x55e2ff30c120 2024-09-21T10:41:34.964 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.fs: PosixFileSystem 2024-09-21T10:41:34.964 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.info_log: 0x55e3010f78e0 2024-09-21T10:41:34.964 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.max_file_opening_threads: 16 2024-09-21T10:41:34.965 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.statistics: (nil) 2024-09-21T10:41:34.965 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.use_fsync: 0 2024-09-21T10:41:34.965 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.max_log_file_size: 0 2024-09-21T10:41:34.965 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-21T10:41:34.965 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-21T10:41:34.965 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.keep_log_file_num: 1000 2024-09-21T10:41:34.965 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.recycle_log_file_num: 0 2024-09-21T10:41:34.965 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.allow_fallocate: 1 2024-09-21T10:41:34.966 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.allow_mmap_reads: 0 2024-09-21T10:41:34.966 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.allow_mmap_writes: 0 2024-09-21T10:41:34.966 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.use_direct_reads: 0 2024-09-21T10:41:34.966 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-21T10:41:34.966 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.create_missing_column_families: 0 2024-09-21T10:41:34.966 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.db_log_dir: 2024-09-21T10:41:34.966 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.wal_dir: 2024-09-21T10:41:34.966 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-21T10:41:34.966 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-21T10:41:34.966 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-21T10:41:34.966 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-21T10:41:34.966 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-21T10:41:34.966 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-21T10:41:34.967 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.advise_random_on_open: 1 2024-09-21T10:41:34.967 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.db_write_buffer_size: 0 2024-09-21T10:41:34.967 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.write_buffer_manager: 0x55e3010fbb80 2024-09-21T10:41:34.967 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-21T10:41:34.967 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-21T10:41:34.967 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-21T10:41:34.967 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.rate_limiter: (nil) 2024-09-21T10:41:34.967 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-21T10:41:34.967 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.wal_recovery_mode: 2 2024-09-21T10:41:34.967 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.enable_thread_tracking: 0 2024-09-21T10:41:34.967 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.enable_pipelined_write: 0 2024-09-21T10:41:34.967 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.unordered_write: 0 2024-09-21T10:41:34.967 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-21T10:41:34.967 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-21T10:41:34.967 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-21T10:41:34.968 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-21T10:41:34.968 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.row_cache: None 2024-09-21T10:41:34.968 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.wal_filter: None 2024-09-21T10:41:34.968 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-21T10:41:34.968 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.allow_ingest_behind: 0 2024-09-21T10:41:34.968 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.two_write_queues: 0 2024-09-21T10:41:34.968 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.manual_wal_flush: 0 2024-09-21T10:41:34.968 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.wal_compression: 0 2024-09-21T10:41:34.968 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.atomic_flush: 0 2024-09-21T10:41:34.968 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-21T10:41:34.968 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-21T10:41:34.968 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-21T10:41:34.968 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.log_readahead_size: 0 2024-09-21T10:41:34.968 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-21T10:41:34.969 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.best_efforts_recovery: 0 2024-09-21T10:41:34.969 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-21T10:41:34.969 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-21T10:41:34.969 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.allow_data_in_errors: 0 2024-09-21T10:41:34.969 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.db_host_id: __hostname__ 2024-09-21T10:41:34.969 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-21T10:41:34.969 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.max_background_jobs: 2 2024-09-21T10:41:34.969 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.max_background_compactions: -1 2024-09-21T10:41:34.969 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.max_subcompactions: 1 2024-09-21T10:41:34.969 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-21T10:41:34.969 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-21T10:41:34.969 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-21T10:41:34.969 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.max_total_wal_size: 0 2024-09-21T10:41:34.969 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-21T10:41:34.970 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-21T10:41:34.970 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-21T10:41:34.970 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-21T10:41:34.970 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.max_open_files: -1 2024-09-21T10:41:34.970 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.bytes_per_sync: 0 2024-09-21T10:41:34.970 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-21T10:41:34.970 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-21T10:41:34.970 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.compaction_readahead_size: 0 2024-09-21T10:41:34.970 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.max_background_flushes: -1 2024-09-21T10:41:34.970 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Compression algorithms supported: 2024-09-21T10:41:34.970 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: kZSTD supported: 0 2024-09-21T10:41:34.970 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: kXpressCompression supported: 0 2024-09-21T10:41:34.970 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: kBZip2Compression supported: 0 2024-09-21T10:41:34.970 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-21T10:41:34.971 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: kLZ4Compression supported: 1 2024-09-21T10:41:34.971 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: kZlibCompression supported: 1 2024-09-21T10:41:34.971 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: kLZ4HCCompression supported: 1 2024-09-21T10:41:34.971 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: kSnappyCompression supported: 1 2024-09-21T10:41:34.971 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-21T10:41:34.971 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-21T10:41:34.971 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi123/store.db/MANIFEST-000015 2024-09-21T10:41:34.971 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-21T10:41:34.971 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-21T10:41:34.971 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.merge_operator: 2024-09-21T10:41:34.971 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.compaction_filter: None 2024-09-21T10:41:34.971 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.compaction_filter_factory: None 2024-09-21T10:41:34.971 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.sst_partitioner_factory: None 2024-09-21T10:41:34.971 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-21T10:41:34.971 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-21T10:41:34.972 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55e3010f7540) 2024-09-21T10:41:34.972 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: cache_index_and_filter_blocks: 1 2024-09-21T10:41:34.972 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-21T10:41:34.972 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-21T10:41:34.972 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: pin_top_level_index_and_filter: 1 2024-09-21T10:41:34.972 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: index_type: 0 2024-09-21T10:41:34.972 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: data_block_index_type: 0 2024-09-21T10:41:34.972 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: index_shortening: 1 2024-09-21T10:41:34.972 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-21T10:41:34.972 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: checksum: 4 2024-09-21T10:41:34.972 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: no_block_cache: 0 2024-09-21T10:41:34.972 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: block_cache: 0x55e3010eb610 2024-09-21T10:41:34.972 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: block_cache_name: BinnedLRUCache 2024-09-21T10:41:34.972 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: block_cache_options: 2024-09-21T10:41:34.972 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: capacity : 536870912 2024-09-21T10:41:34.973 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: num_shard_bits : 4 2024-09-21T10:41:34.973 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: strict_capacity_limit : 0 2024-09-21T10:41:34.973 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: high_pri_pool_ratio: 0.000 2024-09-21T10:41:34.973 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: block_cache_compressed: (nil) 2024-09-21T10:41:34.973 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: persistent_cache: (nil) 2024-09-21T10:41:34.973 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: block_size: 4096 2024-09-21T10:41:34.973 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: block_size_deviation: 10 2024-09-21T10:41:34.973 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: block_restart_interval: 16 2024-09-21T10:41:34.973 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: index_block_restart_interval: 1 2024-09-21T10:41:34.973 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: metadata_block_size: 4096 2024-09-21T10:41:34.973 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: partition_filters: 0 2024-09-21T10:41:34.973 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: use_delta_encoding: 1 2024-09-21T10:41:34.973 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: filter_policy: bloomfilter 2024-09-21T10:41:34.973 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: whole_key_filtering: 1 2024-09-21T10:41:34.973 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: verify_compression: 0 2024-09-21T10:41:34.973 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: read_amp_bytes_per_bit: 0 2024-09-21T10:41:34.974 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: format_version: 5 2024-09-21T10:41:34.974 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: enable_index_compression: 1 2024-09-21T10:41:34.974 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: block_align: 0 2024-09-21T10:41:34.974 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: max_auto_readahead_size: 262144 2024-09-21T10:41:34.974 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: prepopulate_block_cache: 0 2024-09-21T10:41:34.974 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: initial_auto_readahead_size: 8192 2024-09-21T10:41:34.974 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: num_file_reads_for_auto_readahead: 2 2024-09-21T10:41:34.974 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.write_buffer_size: 33554432 2024-09-21T10:41:34.974 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.max_write_buffer_number: 2 2024-09-21T10:41:34.974 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.compression: NoCompression 2024-09-21T10:41:34.974 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.bottommost_compression: Disabled 2024-09-21T10:41:34.974 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.prefix_extractor: nullptr 2024-09-21T10:41:34.974 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-21T10:41:34.974 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.num_levels: 7 2024-09-21T10:41:34.974 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-21T10:41:34.975 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-21T10:41:34.975 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-21T10:41:34.975 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-21T10:41:34.975 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-21T10:41:34.975 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-21T10:41:34.975 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-21T10:41:34.975 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-21T10:41:34.975 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-21T10:41:34.975 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-21T10:41:34.975 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-21T10:41:34.975 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-21T10:41:34.975 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-21T10:41:34.975 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.compression_opts.level: 32767 2024-09-21T10:41:34.975 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.compression_opts.strategy: 0 2024-09-21T10:41:34.975 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-21T10:41:34.976 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-21T10:41:34.976 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-21T10:41:34.976 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-21T10:41:34.976 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.compression_opts.enabled: false 2024-09-21T10:41:34.976 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-21T10:41:34.976 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-21T10:41:34.976 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-21T10:41:34.976 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-21T10:41:34.976 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.target_file_size_base: 67108864 2024-09-21T10:41:34.976 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-21T10:41:34.976 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-21T10:41:34.976 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-21T10:41:34.976 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-21T10:41:34.976 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-21T10:41:34.976 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-21T10:41:34.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-21T10:41:34.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-21T10:41:34.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-21T10:41:34.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-21T10:41:34.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-21T10:41:34.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-21T10:41:34.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-21T10:41:34.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-21T10:41:34.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.arena_block_size: 1048576 2024-09-21T10:41:34.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-21T10:41:34.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-21T10:41:34.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.disable_auto_compactions: 0 2024-09-21T10:41:34.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-21T10:41:34.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-21T10:41:34.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-21T10:41:34.977 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-21T10:41:34.978 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-21T10:41:34.978 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-21T10:41:34.978 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-21T10:41:34.978 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-21T10:41:34.978 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-21T10:41:34.978 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-21T10:41:34.978 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-21T10:41:34.978 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.inplace_update_support: 0 2024-09-21T10:41:34.978 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-21T10:41:34.978 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-21T10:41:34.978 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-21T10:41:34.978 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-21T10:41:34.978 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.bloom_locality: 0 2024-09-21T10:41:34.978 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.max_successive_merges: 0 2024-09-21T10:41:34.979 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-21T10:41:34.979 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.paranoid_file_checks: 0 2024-09-21T10:41:34.979 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.force_consistency_checks: 1 2024-09-21T10:41:34.979 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.report_bg_io_stats: 0 2024-09-21T10:41:34.979 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.ttl: 2592000 2024-09-21T10:41:34.979 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-21T10:41:34.979 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-21T10:41:34.979 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-21T10:41:34.979 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.enable_blob_files: false 2024-09-21T10:41:34.979 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.min_blob_size: 0 2024-09-21T10:41:34.979 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.blob_file_size: 268435456 2024-09-21T10:41:34.979 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-21T10:41:34.979 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-21T10:41:34.979 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-21T10:41:34.979 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-21T10:41:34.980 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-21T10:41:34.980 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.blob_file_starting_level: 0 2024-09-21T10:41:34.980 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-21T10:41:34.980 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 30.sst 2024-09-21T10:41:34.980 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-21T10:41:34.980 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi123/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 32, last_sequence is 11868, log_number is 28,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-21T10:41:34.980 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 28 2024-09-21T10:41:34.980 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 54181fdc-f81b-46d5-93de-b9b3d753f2c8 2024-09-21T10:41:34.980 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726915294926773, "job": 1, "event": "recovery_started", "wal_files": [28]} 2024-09-21T10:41:34.980 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #28 mode 2 2024-09-21T10:41:34.980 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726915294936245, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 33, "file_size": 2263023, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 11869, "largest_seqno": 12881, "table_properties": {"data_size": 2257734, "index_size": 3092, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1157, "raw_key_size": 10183, "raw_average_key_size": 23, "raw_value_size": 2248077, "raw_average_value_size": 5120, "num_data_blocks": 142, "num_entries": 439, "num_filter_entries": 439, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726915294, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "54181fdc-f81b-46d5-93de-b9b3d753f2c8", "db_session_id": "ET1QPGT5D6U09IP9RBK3", "orig_file_number": 33, "seqno_to_time_mapping": "N/A"}} 2024-09-21T10:41:34.980 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726915294937057, "job": 1, "event": "recovery_finished"} 2024-09-21T10:41:34.980 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: [db/version_set.cc:5047] Creating manifest 35 2024-09-21T10:41:34.980 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-21T10:41:34.980 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-smithi123/store.db/000028.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-21T10:41:34.981 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55e30111ae00 2024-09-21T10:41:34.981 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: DB pointer 0x55e30121c000 2024-09-21T10:41:34.981 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-21T10:41:34.981 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-21T10:41:34.981 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: ** DB Stats ** 2024-09-21T10:41:34.981 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T10:41:34.981 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-21T10:41:34.981 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-21T10:41:34.981 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T10:41:34.981 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-21T10:41:34.981 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-21T10:41:34.981 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T10:41:34.981 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: 2024-09-21T10:41:34.981 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: ** Compaction Stats [default] ** 2024-09-21T10:41:34.981 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-21T10:41:34.981 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-21T10:41:34.982 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: L0 1/0 2.16 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 427.2 0.01 0.00 1 0.005 0 0 0.0 0.0 2024-09-21T10:41:34.982 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: L6 1/0 6.69 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-21T10:41:34.982 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Sum 2/0 8.85 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 427.2 0.01 0.00 1 0.005 0 0 0.0 0.0 2024-09-21T10:41:34.982 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 427.2 0.01 0.00 1 0.005 0 0 0.0 0.0 2024-09-21T10:41:34.982 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: 2024-09-21T10:41:34.982 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: ** Compaction Stats [default] ** 2024-09-21T10:41:34.982 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-21T10:41:34.982 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-21T10:41:34.982 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 427.2 0.01 0.00 1 0.005 0 0 0.0 0.0 2024-09-21T10:41:34.982 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: 2024-09-21T10:41:34.983 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-21T10:41:34.983 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: 2024-09-21T10:41:34.983 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T10:41:34.983 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Flush(GB): cumulative 0.002, interval 0.002 2024-09-21T10:41:34.983 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-21T10:41:34.983 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-21T10:41:34.983 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-21T10:41:34.983 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-21T10:41:34.983 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Cumulative compaction: 0.00 GB write, 159.05 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T10:41:34.983 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Interval compaction: 0.00 GB write, 159.05 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T10:41:34.983 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-21T10:41:34.983 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Block cache BinnedLRUCache@0x55e3010eb610#2 capacity: 512.00 MB usage: 4.61 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.4e-05 secs_since: 0 2024-09-21T10:41:34.983 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.28 KB,0.000244379%) IndexBlock(1,3.33 KB,0.000634789%) Misc(1,0.00 KB,0%) 2024-09-21T10:41:34.983 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: 2024-09-21T10:41:34.984 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-21T10:41:34.984 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: starting mon.smithi123 rank 0 at public addrs [v2:172.21.15.123:3300/0,v1:172.21.15.123:6789/0] at bind addrs [v2:172.21.15.123:3300/0,v1:172.21.15.123:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi123 fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 2024-09-21T10:41:34.984 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: mon.smithi123@-1(???) e2 preinit fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 2024-09-21T10:41:34.984 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: mon.smithi123@-1(???).mds e6 new map 2024-09-21T10:41:34.984 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: mon.smithi123@-1(???).mds e6 print_map 2024-09-21T10:41:34.984 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: e6 2024-09-21T10:41:34.984 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-21T10:41:34.984 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-21T10:41:34.984 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-21T10:41:34.984 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: legacy client fscid: 1 2024-09-21T10:41:34.984 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: 2024-09-21T10:41:34.984 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Filesystem 'foofs' (1) 2024-09-21T10:41:34.984 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: fs_name foofs 2024-09-21T10:41:34.984 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: epoch 4 2024-09-21T10:41:34.984 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-21T10:41:34.985 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: created 2024-09-21T10:36:25.765834+0000 2024-09-21T10:41:34.985 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: modified 2024-09-21T10:36:31.639311+0000 2024-09-21T10:41:34.985 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: tableserver 0 2024-09-21T10:41:34.985 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: root 0 2024-09-21T10:41:34.985 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: session_timeout 60 2024-09-21T10:41:34.985 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: session_autoclose 300 2024-09-21T10:41:34.985 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: max_file_size 1099511627776 2024-09-21T10:41:34.985 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: max_xattr_size 65536 2024-09-21T10:41:34.985 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: required_client_features {} 2024-09-21T10:41:34.985 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: last_failure 0 2024-09-21T10:41:34.985 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: last_failure_osd_epoch 0 2024-09-21T10:41:34.985 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-21T10:41:34.985 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: max_mds 1 2024-09-21T10:41:34.985 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: in 0 2024-09-21T10:41:34.985 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: up {0=14492} 2024-09-21T10:41:34.986 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: failed 2024-09-21T10:41:34.986 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: damaged 2024-09-21T10:41:34.986 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: stopped 2024-09-21T10:41:34.986 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: data_pools [3] 2024-09-21T10:41:34.986 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: metadata_pool 2 2024-09-21T10:41:34.986 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: inline_data disabled 2024-09-21T10:41:34.986 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: balancer 2024-09-21T10:41:34.986 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: bal_rank_mask -1 2024-09-21T10:41:34.986 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: standby_count_wanted 1 2024-09-21T10:41:34.986 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: qdb_cluster leader: 0 members: 2024-09-21T10:41:34.986 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: [mds.foofs.smithi123.uqoyow{0:14492} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.123:6834/625956046,v1:172.21.15.123:6835/625956046] compat {c=[1],r=[1],i=[7ff]}] 2024-09-21T10:41:34.986 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: 2024-09-21T10:41:34.986 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: 2024-09-21T10:41:34.987 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Standby daemons: 2024-09-21T10:41:34.987 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: 2024-09-21T10:41:34.987 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: [mds.foofs.smithi175.jrlevw{-1:24269} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.175:6832/3454320220,v1:172.21.15.175:6833/3454320220] compat {c=[1],r=[1],i=[7ff]}] 2024-09-21T10:41:34.987 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: mon.smithi123@-1(???).osd e75 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-21T10:41:34.987 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: mon.smithi123@-1(???).osd e75 crush map has features 432629239337189376, adjusting msgr requires 2024-09-21T10:41:34.987 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: mon.smithi123@-1(???).osd e75 crush map has features 432629239337189376, adjusting msgr requires 2024-09-21T10:41:34.987 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: mon.smithi123@-1(???).osd e75 crush map has features 432629239337189376, adjusting msgr requires 2024-09-21T10:41:34.987 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 ceph-mon[63856]: mon.smithi123@-1(???).paxosservice(auth 1..29) refresh upgraded, format 0 -> 3 2024-09-21T10:41:35.026 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 bash[63838]: 6b95207a2d807be3959310629d20a58fbce105e32dffa46adcf09109666a4896 2024-09-21T10:41:35.026 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:34 smithi123 systemd[1]: Started Ceph mon.smithi123 for 3ccd15e6-7804-11ef-baf6-efdc52797490. 2024-09-21T10:41:35.758 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:35 smithi123 ceph-mon[63856]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mon metadata", "id": "smithi123"}]: dispatch 2024-09-21T10:41:35.758 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:35 smithi123 ceph-mon[63856]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mon metadata", "id": "smithi175"}]: dispatch 2024-09-21T10:41:35.759 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:35 smithi123 ceph-mon[63856]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T10:41:35.759 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:35 smithi123 ceph-mon[63856]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T10:41:35.759 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:35 smithi123 ceph-mon[63856]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T10:41:35.759 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:35 smithi123 ceph-mon[63856]: pgmap v26: 97 pgs: 97 active+clean; 334 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 3.9 MiB/s wr, 393 op/s 2024-09-21T10:41:35.759 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:35 smithi123 ceph-mon[63856]: mon.smithi123 calling monitor election 2024-09-21T10:41:35.759 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:35 smithi123 ceph-mon[63856]: mon.smithi123 is new leader, mons smithi123,smithi175 in quorum (ranks 0,1) 2024-09-21T10:41:35.759 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:35 smithi123 ceph-mon[63856]: monmap epoch 2 2024-09-21T10:41:35.759 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:35 smithi123 ceph-mon[63856]: fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 2024-09-21T10:41:35.759 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:35 smithi123 ceph-mon[63856]: last_changed 2024-09-21T10:31:07.943649+0000 2024-09-21T10:41:35.759 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:35 smithi123 ceph-mon[63856]: created 2024-09-21T10:28:48.510560+0000 2024-09-21T10:41:35.759 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:35 smithi123 ceph-mon[63856]: min_mon_release 17 (quincy) 2024-09-21T10:41:35.759 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:35 smithi123 ceph-mon[63856]: election_strategy: 1 2024-09-21T10:41:35.759 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:35 smithi123 ceph-mon[63856]: 0: [v2:172.21.15.123:3300/0,v1:172.21.15.123:6789/0] mon.smithi123 2024-09-21T10:41:35.760 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:35 smithi123 ceph-mon[63856]: 1: [v2:172.21.15.175:3300/0,v1:172.21.15.175:6789/0] mon.smithi175 2024-09-21T10:41:35.760 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:35 smithi123 ceph-mon[63856]: fsmap foofs:1 {0=foofs.smithi123.uqoyow=up:active} 1 up:standby 2024-09-21T10:41:35.760 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:35 smithi123 ceph-mon[63856]: osdmap e75: 8 total, 8 up, 8 in 2024-09-21T10:41:35.760 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:35 smithi123 ceph-mon[63856]: mgrmap e34: smithi123.stnuit(active, since 48s), standbys: smithi175.txvwrq 2024-09-21T10:41:35.760 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:35 smithi123 ceph-mon[63856]: overall HEALTH_OK 2024-09-21T10:41:35.816 INFO:teuthology.orchestra.run.smithi123.stdout: 5 121565 10.56 MB/sec execute 156 sec latency 63.192 ms 2024-09-21T10:41:36.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:35 smithi175 ceph-mon[25806]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mon metadata", "id": "smithi123"}]: dispatch 2024-09-21T10:41:36.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:35 smithi175 ceph-mon[25806]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mon metadata", "id": "smithi175"}]: dispatch 2024-09-21T10:41:36.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:35 smithi175 ceph-mon[25806]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T10:41:36.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:35 smithi175 ceph-mon[25806]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T10:41:36.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:35 smithi175 ceph-mon[25806]: from='mgr.14774 172.21.15.123:0/3749269913' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T10:41:36.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:35 smithi175 ceph-mon[25806]: pgmap v26: 97 pgs: 97 active+clean; 334 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 3.9 MiB/s wr, 393 op/s 2024-09-21T10:41:36.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:35 smithi175 ceph-mon[25806]: mon.smithi123 calling monitor election 2024-09-21T10:41:36.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:35 smithi175 ceph-mon[25806]: mon.smithi123 is new leader, mons smithi123,smithi175 in quorum (ranks 0,1) 2024-09-21T10:41:36.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:35 smithi175 ceph-mon[25806]: monmap epoch 2 2024-09-21T10:41:36.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:35 smithi175 ceph-mon[25806]: fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 2024-09-21T10:41:36.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:35 smithi175 ceph-mon[25806]: last_changed 2024-09-21T10:31:07.943649+0000 2024-09-21T10:41:36.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:35 smithi175 ceph-mon[25806]: created 2024-09-21T10:28:48.510560+0000 2024-09-21T10:41:36.125 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:35 smithi175 ceph-mon[25806]: min_mon_release 17 (quincy) 2024-09-21T10:41:36.125 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:35 smithi175 ceph-mon[25806]: election_strategy: 1 2024-09-21T10:41:36.125 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:35 smithi175 ceph-mon[25806]: 0: [v2:172.21.15.123:3300/0,v1:172.21.15.123:6789/0] mon.smithi123 2024-09-21T10:41:36.125 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:35 smithi175 ceph-mon[25806]: 1: [v2:172.21.15.175:3300/0,v1:172.21.15.175:6789/0] mon.smithi175 2024-09-21T10:41:36.125 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:35 smithi175 ceph-mon[25806]: fsmap foofs:1 {0=foofs.smithi123.uqoyow=up:active} 1 up:standby 2024-09-21T10:41:36.125 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:35 smithi175 ceph-mon[25806]: osdmap e75: 8 total, 8 up, 8 in 2024-09-21T10:41:36.125 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:35 smithi175 ceph-mon[25806]: mgrmap e34: smithi123.stnuit(active, since 48s), standbys: smithi175.txvwrq 2024-09-21T10:41:36.125 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:35 smithi175 ceph-mon[25806]: overall HEALTH_OK 2024-09-21T10:41:36.780 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:36 smithi123 ceph-mon[63856]: from='mgr.14774 ' entity='' 2024-09-21T10:41:36.780 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:36 smithi123 ceph-mon[63856]: mgrmap e35: smithi123.stnuit(active, since 48s), standbys: smithi175.txvwrq 2024-09-21T10:41:36.780 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:36 smithi123 ceph-mon[63856]: mgrmap e36: smithi123.stnuit(active, since 49s), standbys: smithi175.txvwrq 2024-09-21T10:41:36.816 INFO:teuthology.orchestra.run.smithi123.stdout: 5 122151 10.58 MB/sec execute 157 sec latency 46.059 ms 2024-09-21T10:41:37.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:36 smithi175 ceph-mon[25806]: from='mgr.14774 ' entity='' 2024-09-21T10:41:37.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:36 smithi175 ceph-mon[25806]: mgrmap e35: smithi123.stnuit(active, since 48s), standbys: smithi175.txvwrq 2024-09-21T10:41:37.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:36 smithi175 ceph-mon[25806]: mgrmap e36: smithi123.stnuit(active, since 49s), standbys: smithi175.txvwrq 2024-09-21T10:41:37.816 INFO:teuthology.orchestra.run.smithi123.stdout: 5 122731 10.61 MB/sec execute 158 sec latency 46.174 ms 2024-09-21T10:41:38.816 INFO:teuthology.orchestra.run.smithi123.stdout: 5 123099 10.59 MB/sec execute 159 sec latency 52.444 ms 2024-09-21T10:41:39.816 INFO:teuthology.orchestra.run.smithi123.stdout: 5 123402 10.58 MB/sec execute 160 sec latency 66.178 ms 2024-09-21T10:41:40.816 INFO:teuthology.orchestra.run.smithi123.stdout: 5 123620 10.56 MB/sec execute 161 sec latency 77.867 ms 2024-09-21T10:41:41.816 INFO:teuthology.orchestra.run.smithi123.stdout: 5 123959 10.59 MB/sec execute 162 sec latency 44.134 ms 2024-09-21T10:41:42.398 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:42 smithi123 ceph-mon[63856]: Active manager daemon smithi123.stnuit restarted 2024-09-21T10:41:42.398 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:42 smithi123 ceph-mon[63856]: Activating manager daemon smithi123.stnuit 2024-09-21T10:41:42.398 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:42 smithi123 ceph-mon[63856]: osdmap e76: 8 total, 8 up, 8 in 2024-09-21T10:41:42.398 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:42 smithi123 ceph-mon[63856]: mgrmap e37: smithi123.stnuit(active, starting, since 0.00716563s), standbys: smithi175.txvwrq 2024-09-21T10:41:42.398 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:42 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mon metadata", "id": "smithi123"}]: dispatch 2024-09-21T10:41:42.398 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:42 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mon metadata", "id": "smithi175"}]: dispatch 2024-09-21T10:41:42.399 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:42 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi123.uqoyow"}]: dispatch 2024-09-21T10:41:42.399 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:42 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi175.jrlevw"}]: dispatch 2024-09-21T10:41:42.452 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:42 smithi175 ceph-mon[25806]: Active manager daemon smithi123.stnuit restarted 2024-09-21T10:41:42.452 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:42 smithi175 ceph-mon[25806]: Activating manager daemon smithi123.stnuit 2024-09-21T10:41:42.452 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:42 smithi175 ceph-mon[25806]: osdmap e76: 8 total, 8 up, 8 in 2024-09-21T10:41:42.453 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:42 smithi175 ceph-mon[25806]: mgrmap e37: smithi123.stnuit(active, starting, since 0.00716563s), standbys: smithi175.txvwrq 2024-09-21T10:41:42.453 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:42 smithi175 ceph-mon[25806]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mon metadata", "id": "smithi123"}]: dispatch 2024-09-21T10:41:42.453 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:42 smithi175 ceph-mon[25806]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mon metadata", "id": "smithi175"}]: dispatch 2024-09-21T10:41:42.453 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:42 smithi175 ceph-mon[25806]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi123.uqoyow"}]: dispatch 2024-09-21T10:41:42.453 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:42 smithi175 ceph-mon[25806]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi175.jrlevw"}]: dispatch 2024-09-21T10:41:42.816 INFO:teuthology.orchestra.run.smithi123.stdout: 5 124258 10.57 MB/sec execute 163 sec latency 59.821 ms 2024-09-21T10:41:43.433 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:43 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mgr metadata", "who": "smithi123.stnuit", "id": "smithi123.stnuit"}]: dispatch 2024-09-21T10:41:43.433 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:43 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mgr metadata", "who": "smithi175.txvwrq", "id": "smithi175.txvwrq"}]: dispatch 2024-09-21T10:41:43.433 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:43 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T10:41:43.433 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:43 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T10:41:43.433 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:43 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T10:41:43.433 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:43 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T10:41:43.433 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:43 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T10:41:43.433 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:43 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T10:41:43.433 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:43 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T10:41:43.433 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:43 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T10:41:43.434 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:43 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T10:41:43.434 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:43 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T10:41:43.434 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:43 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T10:41:43.434 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:43 smithi123 ceph-mon[63856]: Manager daemon smithi123.stnuit is now available 2024-09-21T10:41:43.434 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:43 smithi123 ceph-mon[63856]: from='mgr.? 172.21.15.175:0/1207001743' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi175.txvwrq/crt"}]: dispatch 2024-09-21T10:41:43.434 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:43 smithi123 ceph-mon[63856]: from='mgr.? 172.21.15.175:0/1207001743' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T10:41:43.434 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:43 smithi123 ceph-mon[63856]: from='mgr.? 172.21.15.175:0/1207001743' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi175.txvwrq/key"}]: dispatch 2024-09-21T10:41:43.434 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:43 smithi123 ceph-mon[63856]: from='mgr.? 172.21.15.175:0/1207001743' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T10:41:43.434 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:43 smithi123 ceph-mon[63856]: Standby manager daemon smithi175.txvwrq restarted 2024-09-21T10:41:43.434 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:43 smithi123 ceph-mon[63856]: Standby manager daemon smithi175.txvwrq started 2024-09-21T10:41:43.434 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:43 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:41:43.434 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:43 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T10:41:43.434 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:43 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.stnuit/mirror_snapshot_schedule"}]: dispatch 2024-09-21T10:41:43.434 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:43 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T10:41:43.435 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:43 smithi123 ceph-mon[63856]: from='mgr.34102 ' entity='mgr.smithi123.stnuit' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.stnuit/mirror_snapshot_schedule"} : dispatch 2024-09-21T10:41:43.435 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:43 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.stnuit/trash_purge_schedule"}]: dispatch 2024-09-21T10:41:43.435 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:43 smithi123 ceph-mon[63856]: from='mgr.34102 ' entity='mgr.smithi123.stnuit' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.stnuit/trash_purge_schedule"} : dispatch 2024-09-21T10:41:43.512 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:43 smithi175 ceph-mon[25806]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mgr metadata", "who": "smithi123.stnuit", "id": "smithi123.stnuit"}]: dispatch 2024-09-21T10:41:43.512 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:43 smithi175 ceph-mon[25806]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mgr metadata", "who": "smithi175.txvwrq", "id": "smithi175.txvwrq"}]: dispatch 2024-09-21T10:41:43.512 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:43 smithi175 ceph-mon[25806]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T10:41:43.512 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:43 smithi175 ceph-mon[25806]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T10:41:43.512 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:43 smithi175 ceph-mon[25806]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T10:41:43.513 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:43 smithi175 ceph-mon[25806]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T10:41:43.513 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:43 smithi175 ceph-mon[25806]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T10:41:43.513 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:43 smithi175 ceph-mon[25806]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T10:41:43.513 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:43 smithi175 ceph-mon[25806]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T10:41:43.513 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:43 smithi175 ceph-mon[25806]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T10:41:43.513 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:43 smithi175 ceph-mon[25806]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T10:41:43.513 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:43 smithi175 ceph-mon[25806]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T10:41:43.513 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:43 smithi175 ceph-mon[25806]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T10:41:43.513 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:43 smithi175 ceph-mon[25806]: Manager daemon smithi123.stnuit is now available 2024-09-21T10:41:43.513 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:43 smithi175 ceph-mon[25806]: from='mgr.? 172.21.15.175:0/1207001743' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi175.txvwrq/crt"}]: dispatch 2024-09-21T10:41:43.513 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:43 smithi175 ceph-mon[25806]: from='mgr.? 172.21.15.175:0/1207001743' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T10:41:43.513 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:43 smithi175 ceph-mon[25806]: from='mgr.? 172.21.15.175:0/1207001743' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi175.txvwrq/key"}]: dispatch 2024-09-21T10:41:43.514 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:43 smithi175 ceph-mon[25806]: from='mgr.? 172.21.15.175:0/1207001743' entity='mgr.smithi175.txvwrq' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T10:41:43.514 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:43 smithi175 ceph-mon[25806]: Standby manager daemon smithi175.txvwrq restarted 2024-09-21T10:41:43.514 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:43 smithi175 ceph-mon[25806]: Standby manager daemon smithi175.txvwrq started 2024-09-21T10:41:43.514 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:43 smithi175 ceph-mon[25806]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:41:43.514 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:43 smithi175 ceph-mon[25806]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T10:41:43.514 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:43 smithi175 ceph-mon[25806]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.stnuit/mirror_snapshot_schedule"}]: dispatch 2024-09-21T10:41:43.514 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:43 smithi175 ceph-mon[25806]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T10:41:43.514 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:43 smithi175 ceph-mon[25806]: from='mgr.34102 ' entity='mgr.smithi123.stnuit' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.stnuit/mirror_snapshot_schedule"} : dispatch 2024-09-21T10:41:43.514 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:43 smithi175 ceph-mon[25806]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.stnuit/trash_purge_schedule"}]: dispatch 2024-09-21T10:41:43.514 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:43 smithi175 ceph-mon[25806]: from='mgr.34102 ' entity='mgr.smithi123.stnuit' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi123.stnuit/trash_purge_schedule"} : dispatch 2024-09-21T10:41:43.816 INFO:teuthology.orchestra.run.smithi123.stdout: 5 124480 10.57 MB/sec execute 164 sec latency 66.413 ms 2024-09-21T10:41:44.231 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:44 smithi123 ceph-mon[63856]: mgrmap e38: smithi123.stnuit(active, since 1.08412s), standbys: smithi175.txvwrq 2024-09-21T10:41:44.231 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:44 smithi123 ceph-mon[63856]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T10:41:44.231 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:44 smithi123 ceph-mon[63856]: pgmap v3: 97 pgs: 97 active+clean; 345 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T10:41:44.595 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:44 smithi175 ceph-mon[25806]: mgrmap e38: smithi123.stnuit(active, since 1.08412s), standbys: smithi175.txvwrq 2024-09-21T10:41:44.595 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:44 smithi175 ceph-mon[25806]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T10:41:44.595 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:44 smithi175 ceph-mon[25806]: pgmap v3: 97 pgs: 97 active+clean; 345 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T10:41:44.817 INFO:teuthology.orchestra.run.smithi123.stdout: 5 124681 10.53 MB/sec execute 165 sec latency 77.728 ms 2024-09-21T10:41:45.242 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:45 smithi123 ceph-mon[63856]: pgmap v4: 97 pgs: 97 active+clean; 345 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T10:41:45.242 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:45 smithi123 ceph-mon[63856]: mgrmap e39: smithi123.stnuit(active, since 2s), standbys: smithi175.txvwrq 2024-09-21T10:41:45.574 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:45 smithi175 ceph-mon[25806]: pgmap v4: 97 pgs: 97 active+clean; 345 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T10:41:45.574 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:45 smithi175 ceph-mon[25806]: mgrmap e39: smithi123.stnuit(active, since 2s), standbys: smithi175.txvwrq 2024-09-21T10:41:45.816 INFO:teuthology.orchestra.run.smithi123.stdout: 5 124935 10.49 MB/sec execute 166 sec latency 62.143 ms 2024-09-21T10:41:46.522 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:46 smithi123 ceph-mon[63856]: [21/Sep/2024:10:41:44] ENGINE Bus STARTING 2024-09-21T10:41:46.522 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:46 smithi123 ceph-mon[63856]: [21/Sep/2024:10:41:44] ENGINE Serving on https://172.21.15.123:7150 2024-09-21T10:41:46.522 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:46 smithi123 ceph-mon[63856]: [21/Sep/2024:10:41:44] ENGINE Error in HTTPServer.serve 2024-09-21T10:41:46.522 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: Traceback (most recent call last): 2024-09-21T10:41:46.523 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T10:41:46.523 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: self._connections.run(self.expiration_interval) 2024-09-21T10:41:46.523 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T10:41:46.523 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: self._run(expiration_interval) 2024-09-21T10:41:46.523 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T10:41:46.523 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T10:41:46.523 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T10:41:46.523 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T10:41:46.523 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T10:41:46.523 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: s = self.context.wrap_socket( 2024-09-21T10:41:46.523 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T10:41:46.523 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: return self.sslsocket_class._create( 2024-09-21T10:41:46.523 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T10:41:46.524 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: self.do_handshake() 2024-09-21T10:41:46.524 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T10:41:46.524 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: self._sslobj.do_handshake() 2024-09-21T10:41:46.524 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T10:41:46.524 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:46 smithi123 ceph-mon[63856]: [21/Sep/2024:10:41:45] ENGINE Serving on http://172.21.15.123:8765 2024-09-21T10:41:46.524 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:46 smithi123 ceph-mon[63856]: [21/Sep/2024:10:41:45] ENGINE Bus STARTED 2024-09-21T10:41:46.578 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:46 smithi175 ceph-mon[25806]: [21/Sep/2024:10:41:44] ENGINE Bus STARTING 2024-09-21T10:41:46.578 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:46 smithi175 ceph-mon[25806]: [21/Sep/2024:10:41:44] ENGINE Serving on https://172.21.15.123:7150 2024-09-21T10:41:46.579 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:46 smithi175 ceph-mon[25806]: [21/Sep/2024:10:41:44] ENGINE Error in HTTPServer.serve 2024-09-21T10:41:46.579 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: Traceback (most recent call last): 2024-09-21T10:41:46.579 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T10:41:46.579 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: self._connections.run(self.expiration_interval) 2024-09-21T10:41:46.579 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T10:41:46.579 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: self._run(expiration_interval) 2024-09-21T10:41:46.579 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T10:41:46.579 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T10:41:46.579 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T10:41:46.579 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T10:41:46.579 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T10:41:46.580 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: s = self.context.wrap_socket( 2024-09-21T10:41:46.580 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T10:41:46.580 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: return self.sslsocket_class._create( 2024-09-21T10:41:46.581 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T10:41:46.581 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: self.do_handshake() 2024-09-21T10:41:46.581 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T10:41:46.581 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: self._sslobj.do_handshake() 2024-09-21T10:41:46.581 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T10:41:46.581 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:46 smithi175 ceph-mon[25806]: [21/Sep/2024:10:41:45] ENGINE Serving on http://172.21.15.123:8765 2024-09-21T10:41:46.582 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:46 smithi175 ceph-mon[25806]: [21/Sep/2024:10:41:45] ENGINE Bus STARTED 2024-09-21T10:41:46.816 INFO:teuthology.orchestra.run.smithi123.stdout: 5 125279 10.45 MB/sec execute 167 sec latency 75.743 ms 2024-09-21T10:41:47.546 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:47 smithi123 ceph-mon[63856]: pgmap v5: 97 pgs: 97 active+clean; 345 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T10:41:47.546 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:47 smithi123 ceph-mon[63856]: mgrmap e40: smithi123.stnuit(active, since 4s), standbys: smithi175.txvwrq 2024-09-21T10:41:47.547 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:47 smithi123 ceph-mon[63856]: from='mgr.34102 ' entity='mgr.smithi123.stnuit' 2024-09-21T10:41:47.547 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:47 smithi123 ceph-mon[63856]: from='mgr.34102 ' entity='mgr.smithi123.stnuit' 2024-09-21T10:41:47.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:47 smithi175 ceph-mon[25806]: pgmap v5: 97 pgs: 97 active+clean; 345 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T10:41:47.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:47 smithi175 ceph-mon[25806]: mgrmap e40: smithi123.stnuit(active, since 4s), standbys: smithi175.txvwrq 2024-09-21T10:41:47.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:47 smithi175 ceph-mon[25806]: from='mgr.34102 ' entity='mgr.smithi123.stnuit' 2024-09-21T10:41:47.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:47 smithi175 ceph-mon[25806]: from='mgr.34102 ' entity='mgr.smithi123.stnuit' 2024-09-21T10:41:47.816 INFO:teuthology.orchestra.run.smithi123.stdout: 5 125520 10.42 MB/sec execute 168 sec latency 125.381 ms 2024-09-21T10:41:48.814 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:48 smithi123 ceph-mon[63856]: from='mgr.34102 ' entity='mgr.smithi123.stnuit' 2024-09-21T10:41:48.814 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:48 smithi123 ceph-mon[63856]: from='mgr.34102 ' entity='mgr.smithi123.stnuit' 2024-09-21T10:41:48.816 INFO:teuthology.orchestra.run.smithi123.stdout: 5 125825 10.41 MB/sec execute 169 sec latency 55.842 ms 2024-09-21T10:41:48.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:48 smithi175 ceph-mon[25806]: from='mgr.34102 ' entity='mgr.smithi123.stnuit' 2024-09-21T10:41:48.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:48 smithi175 ceph-mon[25806]: from='mgr.34102 ' entity='mgr.smithi123.stnuit' 2024-09-21T10:41:49.817 INFO:teuthology.orchestra.run.smithi123.stdout: 5 126301 10.42 MB/sec execute 170 sec latency 45.535 ms 2024-09-21T10:41:49.837 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:49 smithi123 ceph-mon[63856]: pgmap v6: 97 pgs: 97 active+clean; 345 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T10:41:49.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:49 smithi175 ceph-mon[25806]: pgmap v6: 97 pgs: 97 active+clean; 345 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-21T10:41:50.816 INFO:teuthology.orchestra.run.smithi123.stdout: 5 126929 10.45 MB/sec execute 171 sec latency 24.835 ms 2024-09-21T10:41:51.817 INFO:teuthology.orchestra.run.smithi123.stdout: 5 127129 10.42 MB/sec execute 172 sec latency 70.583 ms 2024-09-21T10:41:51.857 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:51 smithi123 ceph-mon[63856]: pgmap v7: 97 pgs: 97 active+clean; 342 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 32 KiB/s rd, 2.7 MiB/s wr, 315 op/s 2024-09-21T10:41:51.857 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:51 smithi123 ceph-mon[63856]: from='mgr.34102 ' entity='mgr.smithi123.stnuit' 2024-09-21T10:41:51.857 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:51 smithi123 ceph-mon[63856]: from='mgr.34102 ' entity='mgr.smithi123.stnuit' 2024-09-21T10:41:51.857 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:51 smithi123 ceph-mon[63856]: from='mgr.34102 ' entity='mgr.smithi123.stnuit' 2024-09-21T10:41:51.858 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:51 smithi123 ceph-mon[63856]: from='mgr.34102 ' entity='mgr.smithi123.stnuit' 2024-09-21T10:41:51.858 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:51 smithi123 ceph-mon[63856]: from='mgr.34102 ' entity='mgr.smithi123.stnuit' 2024-09-21T10:41:51.858 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:51 smithi123 ceph-mon[63856]: from='mgr.34102 ' entity='mgr.smithi123.stnuit' 2024-09-21T10:41:51.858 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:51 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:41:51.858 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:51 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T10:41:52.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:51 smithi175 ceph-mon[25806]: pgmap v7: 97 pgs: 97 active+clean; 342 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 32 KiB/s rd, 2.7 MiB/s wr, 315 op/s 2024-09-21T10:41:52.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:51 smithi175 ceph-mon[25806]: from='mgr.34102 ' entity='mgr.smithi123.stnuit' 2024-09-21T10:41:52.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:51 smithi175 ceph-mon[25806]: from='mgr.34102 ' entity='mgr.smithi123.stnuit' 2024-09-21T10:41:52.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:51 smithi175 ceph-mon[25806]: from='mgr.34102 ' entity='mgr.smithi123.stnuit' 2024-09-21T10:41:52.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:51 smithi175 ceph-mon[25806]: from='mgr.34102 ' entity='mgr.smithi123.stnuit' 2024-09-21T10:41:52.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:51 smithi175 ceph-mon[25806]: from='mgr.34102 ' entity='mgr.smithi123.stnuit' 2024-09-21T10:41:52.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:51 smithi175 ceph-mon[25806]: from='mgr.34102 ' entity='mgr.smithi123.stnuit' 2024-09-21T10:41:52.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:51 smithi175 ceph-mon[25806]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:41:52.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:51 smithi175 ceph-mon[25806]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T10:41:52.817 INFO:teuthology.orchestra.run.smithi123.stdout: 5 127484 10.45 MB/sec execute 173 sec latency 151.968 ms 2024-09-21T10:41:52.912 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:52 smithi123 ceph-mon[63856]: Updating smithi123:/etc/ceph/ceph.conf 2024-09-21T10:41:52.912 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:52 smithi123 ceph-mon[63856]: Updating smithi175:/etc/ceph/ceph.conf 2024-09-21T10:41:52.912 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:52 smithi123 ceph-mon[63856]: from='mgr.34102 ' entity='mgr.smithi123.stnuit' 2024-09-21T10:41:52.912 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:52 smithi123 ceph-mon[63856]: from='mgr.34102 ' entity='mgr.smithi123.stnuit' 2024-09-21T10:41:52.912 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:52 smithi123 ceph-mon[63856]: from='mgr.34102 ' entity='mgr.smithi123.stnuit' 2024-09-21T10:41:52.912 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:52 smithi123 ceph-mon[63856]: from='mgr.34102 ' entity='mgr.smithi123.stnuit' 2024-09-21T10:41:52.912 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:52 smithi123 ceph-mon[63856]: from='mgr.34102 ' entity='mgr.smithi123.stnuit' 2024-09-21T10:41:52.912 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:52 smithi123 ceph-mon[63856]: from='mgr.34102 ' entity='mgr.smithi123.stnuit' 2024-09-21T10:41:52.913 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:52 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:41:52.913 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:52 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T10:41:52.913 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:52 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-21T10:41:53.095 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:52 smithi175 ceph-mon[25806]: Updating smithi123:/etc/ceph/ceph.conf 2024-09-21T10:41:53.095 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:52 smithi175 ceph-mon[25806]: Updating smithi175:/etc/ceph/ceph.conf 2024-09-21T10:41:53.095 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:52 smithi175 ceph-mon[25806]: from='mgr.34102 ' entity='mgr.smithi123.stnuit' 2024-09-21T10:41:53.095 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:52 smithi175 ceph-mon[25806]: from='mgr.34102 ' entity='mgr.smithi123.stnuit' 2024-09-21T10:41:53.095 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:52 smithi175 ceph-mon[25806]: from='mgr.34102 ' entity='mgr.smithi123.stnuit' 2024-09-21T10:41:53.095 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:52 smithi175 ceph-mon[25806]: from='mgr.34102 ' entity='mgr.smithi123.stnuit' 2024-09-21T10:41:53.095 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:52 smithi175 ceph-mon[25806]: from='mgr.34102 ' entity='mgr.smithi123.stnuit' 2024-09-21T10:41:53.095 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:52 smithi175 ceph-mon[25806]: from='mgr.34102 ' entity='mgr.smithi123.stnuit' 2024-09-21T10:41:53.095 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:52 smithi175 ceph-mon[25806]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T10:41:53.096 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:52 smithi175 ceph-mon[25806]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T10:41:53.096 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:52 smithi175 ceph-mon[25806]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-21T10:41:53.817 INFO:teuthology.orchestra.run.smithi123.stdout: 5 127656 10.42 MB/sec execute 174 sec latency 219.426 ms 2024-09-21T10:41:53.923 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:53 smithi123 ceph-mon[63856]: Updating smithi123:/var/lib/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T10:41:53.923 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:53 smithi123 ceph-mon[63856]: Updating smithi175:/var/lib/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T10:41:53.923 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:53 smithi123 ceph-mon[63856]: Updating smithi123:/etc/ceph/ceph.client.admin.keyring 2024-09-21T10:41:53.923 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:53 smithi123 ceph-mon[63856]: Updating smithi175:/etc/ceph/ceph.client.admin.keyring 2024-09-21T10:41:53.923 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:53 smithi123 ceph-mon[63856]: Updating smithi123:/var/lib/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T10:41:53.923 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:53 smithi123 ceph-mon[63856]: Updating smithi175:/var/lib/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T10:41:53.923 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:53 smithi123 ceph-mon[63856]: pgmap v8: 97 pgs: 97 active+clean; 342 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 24 KiB/s rd, 2.1 MiB/s wr, 244 op/s 2024-09-21T10:41:54.119 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:53 smithi175 ceph-mon[25806]: Updating smithi123:/var/lib/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T10:41:54.119 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:53 smithi175 ceph-mon[25806]: Updating smithi175:/var/lib/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T10:41:54.119 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:53 smithi175 ceph-mon[25806]: Updating smithi123:/etc/ceph/ceph.client.admin.keyring 2024-09-21T10:41:54.119 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:53 smithi175 ceph-mon[25806]: Updating smithi175:/etc/ceph/ceph.client.admin.keyring 2024-09-21T10:41:54.119 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:53 smithi175 ceph-mon[25806]: Updating smithi123:/var/lib/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T10:41:54.119 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:53 smithi175 ceph-mon[25806]: Updating smithi175:/var/lib/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T10:41:54.119 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:53 smithi175 ceph-mon[25806]: pgmap v8: 97 pgs: 97 active+clean; 342 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 24 KiB/s rd, 2.1 MiB/s wr, 244 op/s 2024-09-21T10:41:54.817 INFO:teuthology.orchestra.run.smithi123.stdout: 5 127892 10.40 MB/sec execute 175 sec latency 359.894 ms 2024-09-21T10:41:54.982 INFO:teuthology.orchestra.run.smithi123.stdout:true 2024-09-21T10:41:55.207 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:55 smithi123 ceph-mon[63856]: Upgrade: Updating mon.smithi175 2024-09-21T10:41:55.207 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:55 smithi123 ceph-mon[63856]: from='mgr.34102 ' entity='mgr.smithi123.stnuit' 2024-09-21T10:41:55.207 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:55 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T10:41:55.207 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:55 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T10:41:55.207 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:55 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:41:55.208 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:55 smithi123 ceph-mon[63856]: Deploying daemon mon.smithi175 on smithi175 2024-09-21T10:41:55.208 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:55 smithi123 ceph-mon[63856]: pgmap v9: 97 pgs: 97 active+clean; 347 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 23 KiB/s rd, 4.3 MiB/s wr, 444 op/s 2024-09-21T10:41:55.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:55 smithi175 ceph-mon[25806]: Upgrade: Updating mon.smithi175 2024-09-21T10:41:55.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:55 smithi175 ceph-mon[25806]: from='mgr.34102 ' entity='mgr.smithi123.stnuit' 2024-09-21T10:41:55.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:55 smithi175 ceph-mon[25806]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T10:41:55.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:55 smithi175 ceph-mon[25806]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T10:41:55.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:55 smithi175 ceph-mon[25806]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T10:41:55.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:55 smithi175 ceph-mon[25806]: Deploying daemon mon.smithi175 on smithi175 2024-09-21T10:41:55.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:55 smithi175 ceph-mon[25806]: pgmap v9: 97 pgs: 97 active+clean; 347 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 23 KiB/s rd, 4.3 MiB/s wr, 444 op/s 2024-09-21T10:41:55.798 INFO:teuthology.orchestra.run.smithi123.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T10:41:55.798 INFO:teuthology.orchestra.run.smithi123.stdout:alertmanager.smithi123 smithi123 *:9093,9094 running (2m) 4s ago 11m 19.7M - 0.27.0 11f11916f8cd 5d550bd97de5 2024-09-21T10:41:55.798 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi123 smithi123 running (11m) 4s ago 11m 6405k - 17.2.0 e1d6a67b021e c459f87d5b50 2024-09-21T10:41:55.798 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi175 smithi175 running (10m) 8s ago 10m 6405k - 17.2.0 e1d6a67b021e e35e422d5583 2024-09-21T10:41:55.798 INFO:teuthology.orchestra.run.smithi123.stdout:grafana.smithi123 smithi123 *:3000 running (2m) 4s ago 11m 43.4M - dad864ee21e9 771df81763ba 2024-09-21T10:41:55.798 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi123.uqoyow smithi123 running (5m) 4s ago 5m 40.4M - 17.2.0 e1d6a67b021e f0a21eaac266 2024-09-21T10:41:55.799 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi175.jrlevw smithi175 running (5m) 8s ago 5m 13.1M - 17.2.0 e1d6a67b021e b21b16f4bc50 2024-09-21T10:41:55.799 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi123.stnuit smithi123 *:8443,9283,8765 running (88s) 4s ago 12m 489M - 19.3.0-5103-g5c1cdc3c 8f4d013db793 6812cc5cb56b 2024-09-21T10:41:55.799 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi175.txvwrq smithi175 *:8443,9283,8765 running (37s) 8s ago 10m 452M - 19.3.0-5103-g5c1cdc3c 8f4d013db793 304f0408075e 2024-09-21T10:41:55.799 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi123 smithi123 running (21s) 4s ago 12m 30.6M 2048M 19.3.0-5103-g5c1cdc3c 8f4d013db793 6b95207a2d80 2024-09-21T10:41:55.799 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi175 smithi175 running (10m) 8s ago 10m 52.3M 2048M 17.2.0 e1d6a67b021e 753f564c8cf5 2024-09-21T10:41:55.799 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.0.0.smithi123.ddatpr smithi123 *:2049 running (5m) 4s ago 5m 275M - 4.0 e1d6a67b021e 36f8cb2281eb 2024-09-21T10:41:55.799 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.1.0.smithi175.wpawer smithi175 *:2049 running (5m) 8s ago 5m 89.7M - 4.0 e1d6a67b021e 2200aeed9200 2024-09-21T10:41:55.799 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi123 smithi123 *:9100 running (2m) 4s ago 11m 10.2M - 1.7.0 72c9c2088986 ca7d0b29af81 2024-09-21T10:41:55.799 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi175 smithi175 *:9100 running (98s) 8s ago 10m 11.8M - 1.7.0 72c9c2088986 46c4f6b7c149 2024-09-21T10:41:55.799 INFO:teuthology.orchestra.run.smithi123.stdout:osd.0 smithi123 running (9m) 4s ago 9m 449M 1936M 17.2.0 e1d6a67b021e a94edf88c456 2024-09-21T10:41:55.799 INFO:teuthology.orchestra.run.smithi123.stdout:osd.1 smithi123 running (9m) 4s ago 9m 404M 1936M 17.2.0 e1d6a67b021e 59aee515b0f8 2024-09-21T10:41:55.799 INFO:teuthology.orchestra.run.smithi123.stdout:osd.2 smithi123 running (8m) 4s ago 8m 293M 1936M 17.2.0 e1d6a67b021e 7db93f8ce5b5 2024-09-21T10:41:55.799 INFO:teuthology.orchestra.run.smithi123.stdout:osd.3 smithi123 running (8m) 4s ago 8m 420M 1936M 17.2.0 e1d6a67b021e c1672dea7b1f 2024-09-21T10:41:55.799 INFO:teuthology.orchestra.run.smithi123.stdout:osd.4 smithi175 running (8m) 8s ago 8m 429M 2704M 17.2.0 e1d6a67b021e 48fe728c259e 2024-09-21T10:41:55.800 INFO:teuthology.orchestra.run.smithi123.stdout:osd.5 smithi175 running (7m) 8s ago 7m 486M 2704M 17.2.0 e1d6a67b021e e270d5a21b14 2024-09-21T10:41:55.800 INFO:teuthology.orchestra.run.smithi123.stdout:osd.6 smithi175 running (7m) 8s ago 7m 420M 2704M 17.2.0 e1d6a67b021e 9bcbf018eb20 2024-09-21T10:41:55.800 INFO:teuthology.orchestra.run.smithi123.stdout:osd.7 smithi175 running (6m) 8s ago 6m 561M 2704M 17.2.0 e1d6a67b021e e9b8f1f90994 2024-09-21T10:41:55.800 INFO:teuthology.orchestra.run.smithi123.stdout:prometheus.smithi123 smithi123 *:9095 running (43s) 4s ago 10m 46.0M - 2.51.0 1d3b7f56885b 37f944c9de88 2024-09-21T10:41:55.817 INFO:teuthology.orchestra.run.smithi123.stdout: 5 128163 10.39 MB/sec execute 176 sec latency 67.914 ms 2024-09-21T10:41:56.229 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:56 smithi123 ceph-mon[63856]: from='client.34132 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:41:56.229 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:41:56 smithi123 ceph-mon[63856]: from='client.34136 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:41:56.296 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-09-21T10:41:56.296 INFO:teuthology.orchestra.run.smithi123.stdout: "mon": { 2024-09-21T10:41:56.296 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-21T10:41:56.296 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev)": 1 2024-09-21T10:41:56.296 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:41:56.297 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr": { 2024-09-21T10:41:56.297 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev)": 2 2024-09-21T10:41:56.297 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:41:56.297 INFO:teuthology.orchestra.run.smithi123.stdout: "osd": { 2024-09-21T10:41:56.297 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-21T10:41:56.297 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:41:56.297 INFO:teuthology.orchestra.run.smithi123.stdout: "mds": { 2024-09-21T10:41:56.297 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T10:41:56.297 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:41:56.297 INFO:teuthology.orchestra.run.smithi123.stdout: "overall": { 2024-09-21T10:41:56.297 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 11, 2024-09-21T10:41:56.297 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev)": 3 2024-09-21T10:41:56.297 INFO:teuthology.orchestra.run.smithi123.stdout: } 2024-09-21T10:41:56.297 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-09-21T10:41:56.524 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:56 smithi175 ceph-mon[25806]: from='client.34132 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:41:56.524 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:56 smithi175 ceph-mon[25806]: from='client.34136 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:41:56.698 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-09-21T10:41:56.698 INFO:teuthology.orchestra.run.smithi123.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472", 2024-09-21T10:41:56.698 INFO:teuthology.orchestra.run.smithi123.stdout: "in_progress": true, 2024-09-21T10:41:56.698 INFO:teuthology.orchestra.run.smithi123.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T10:41:56.698 INFO:teuthology.orchestra.run.smithi123.stdout: "services_complete": [ 2024-09-21T10:41:56.698 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr" 2024-09-21T10:41:56.699 INFO:teuthology.orchestra.run.smithi123.stdout: ], 2024-09-21T10:41:56.699 INFO:teuthology.orchestra.run.smithi123.stdout: "progress": "3/23 daemons upgraded", 2024-09-21T10:41:56.699 INFO:teuthology.orchestra.run.smithi123.stdout: "message": "Currently upgrading mon daemons", 2024-09-21T10:41:56.699 INFO:teuthology.orchestra.run.smithi123.stdout: "is_paused": false 2024-09-21T10:41:56.699 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-09-21T10:41:56.817 INFO:teuthology.orchestra.run.smithi123.stdout: 5 128344 10.36 MB/sec execute 177 sec latency 72.377 ms 2024-09-21T10:41:56.840 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:56 smithi175 systemd[1]: Stopping Ceph mon.smithi175 for 3ccd15e6-7804-11ef-baf6-efdc52797490... 2024-09-21T10:41:57.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:56 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-mon-smithi175[25779]: 2024-09-21T10:41:56.836+0000 7fc31d049700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi175 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:41:57.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:56 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-mon-smithi175[25779]: 2024-09-21T10:41:56.836+0000 7fc31d049700 -1 mon.smithi175@1(peon) e2 *** Got Signal Terminated *** 2024-09-21T10:41:57.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:56 smithi175 podman[49147]: 2024-09-21 10:41:56.909473996 +0000 UTC m=+0.190217369 container died 753f564c8cf5303e2bdc9e017672e88d8f8f26c1a1340b0e81120f22aeef1584 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-mon-smithi175, io.openshift.expose-services=, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, architecture=x86_64, ceph=True, com.redhat.component=centos-stream-container, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., name=centos-stream) 2024-09-21T10:41:57.245 INFO:teuthology.orchestra.run.smithi123.stdout:HEALTH_OK 2024-09-21T10:41:57.574 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:57 smithi175 podman[49147]: 2024-09-21 10:41:57.30409426 +0000 UTC m=+0.584837634 container cleanup 753f564c8cf5303e2bdc9e017672e88d8f8f26c1a1340b0e81120f22aeef1584 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-mon-smithi175, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , name=centos-stream, GIT_BRANCH=HEAD, ceph=True, vendor=Red Hat, Inc., io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, distribution-scope=public, version=8, vcs-type=git, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, architecture=x86_64) 2024-09-21T10:41:57.574 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:57 smithi175 bash[49147]: ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-mon-smithi175 2024-09-21T10:41:57.817 INFO:teuthology.orchestra.run.smithi123.stdout: 5 128564 10.33 MB/sec execute 178 sec latency 76.359 ms 2024-09-21T10:41:57.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:57 smithi175 podman[49158]: 2024-09-21 10:41:57.57255296 +0000 UTC m=+0.658573103 container remove 753f564c8cf5303e2bdc9e017672e88d8f8f26c1a1340b0e81120f22aeef1584 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-mon-smithi175, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , vcs-type=git, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, RELEASE=HEAD, architecture=x86_64, name=centos-stream, io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-09-21T10:41:57.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:57 smithi175 systemd[1]: ceph-3ccd15e6-7804-11ef-baf6-efdc52797490@mon.smithi175.service: Deactivated successfully. 2024-09-21T10:41:57.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:57 smithi175 systemd[1]: Stopped Ceph mon.smithi175 for 3ccd15e6-7804-11ef-baf6-efdc52797490. 2024-09-21T10:41:57.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:57 smithi175 systemd[1]: ceph-3ccd15e6-7804-11ef-baf6-efdc52797490@mon.smithi175.service: Consumed 6.911s CPU time. 2024-09-21T10:41:58.267 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 systemd[1]: Starting Ceph mon.smithi175 for 3ccd15e6-7804-11ef-baf6-efdc52797490... 2024-09-21T10:41:58.267 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 podman[49285]: 2024-09-21 10:41:58.131697458 +0000 UTC m=+0.017895020 image pull 8f4d013db793795fef09ff5edf96e6b42ba8cd2c9158b2f0b7a5d7c1e657c063 quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472 2024-09-21T10:41:58.267 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 podman[49285]: 2024-09-21 10:41:58.26574704 +0000 UTC m=+0.151944602 container create 3abee2d37f0f1c7e54993398d30da21d183d47b7a44609532666b16d2f74b204 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-mon-smithi175, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, org.label-schema.license=GPLv2) 2024-09-21T10:41:58.817 INFO:teuthology.orchestra.run.smithi123.stdout: 5 128844 10.30 MB/sec execute 179 sec latency 75.080 ms 2024-09-21T10:41:58.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 podman[49285]: 2024-09-21 10:41:58.535516527 +0000 UTC m=+0.421714089 container init 3abee2d37f0f1c7e54993398d30da21d183d47b7a44609532666b16d2f74b204 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-mon-smithi175, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, io.buildah.version=1.37.2) 2024-09-21T10:41:58.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 podman[49285]: 2024-09-21 10:41:58.540048014 +0000 UTC m=+0.426245578 container start 3abee2d37f0f1c7e54993398d30da21d183d47b7a44609532666b16d2f74b204 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-mon-smithi175, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, ceph=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T10:41:58.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: set uid:gid to 167:167 (ceph:ceph) 2024-09-21T10:41:58.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev), process ceph-mon, pid 2 2024-09-21T10:41:58.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: pidfile_write: ignore empty --pid-file 2024-09-21T10:41:58.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: load: jerasure load: lrc 2024-09-21T10:41:58.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: RocksDB version: 7.9.2 2024-09-21T10:41:58.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Git sha 0 2024-09-21T10:41:58.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Compile date 2024-09-21 08:36:07 2024-09-21T10:41:58.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: DB SUMMARY 2024-09-21T10:41:58.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: DB Session ID: PNTTSECBWNBA16F57BOO 2024-09-21T10:41:58.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: CURRENT file: CURRENT 2024-09-21T10:41:58.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: IDENTITY file: IDENTITY 2024-09-21T10:41:58.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: MANIFEST file: MANIFEST-000009 size: 903 Bytes 2024-09-21T10:41:58.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi175/store.db dir, Total Num: 1, files: 000024.sst 2024-09-21T10:41:58.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi175/store.db: 000022.log size: 6880767 ; 2024-09-21T10:41:58.875 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.error_if_exists: 0 2024-09-21T10:41:58.875 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.create_if_missing: 0 2024-09-21T10:41:58.875 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.paranoid_checks: 1 2024-09-21T10:41:58.875 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-21T10:41:58.875 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-21T10:41:58.875 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-21T10:41:58.875 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.env: 0x563b26076120 2024-09-21T10:41:58.875 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.fs: PosixFileSystem 2024-09-21T10:41:58.875 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.info_log: 0x563b26d538e0 2024-09-21T10:41:58.875 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.max_file_opening_threads: 16 2024-09-21T10:41:58.877 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.statistics: (nil) 2024-09-21T10:41:58.877 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.use_fsync: 0 2024-09-21T10:41:58.877 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.max_log_file_size: 0 2024-09-21T10:41:58.877 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-21T10:41:58.877 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-21T10:41:58.877 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.keep_log_file_num: 1000 2024-09-21T10:41:58.877 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.recycle_log_file_num: 0 2024-09-21T10:41:58.877 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.allow_fallocate: 1 2024-09-21T10:41:58.877 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.allow_mmap_reads: 0 2024-09-21T10:41:58.877 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.allow_mmap_writes: 0 2024-09-21T10:41:58.877 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.use_direct_reads: 0 2024-09-21T10:41:58.877 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-21T10:41:58.877 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.create_missing_column_families: 0 2024-09-21T10:41:58.877 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.db_log_dir: 2024-09-21T10:41:58.878 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.wal_dir: 2024-09-21T10:41:58.878 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-21T10:41:58.878 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-21T10:41:58.878 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-21T10:41:58.878 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-21T10:41:58.878 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-21T10:41:58.878 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-21T10:41:58.878 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.advise_random_on_open: 1 2024-09-21T10:41:58.878 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.db_write_buffer_size: 0 2024-09-21T10:41:58.878 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.write_buffer_manager: 0x563b26d57b80 2024-09-21T10:41:58.878 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-21T10:41:58.878 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-21T10:41:58.878 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-21T10:41:58.879 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.rate_limiter: (nil) 2024-09-21T10:41:58.879 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-21T10:41:58.879 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.wal_recovery_mode: 2 2024-09-21T10:41:58.879 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.enable_thread_tracking: 0 2024-09-21T10:41:58.879 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.enable_pipelined_write: 0 2024-09-21T10:41:58.879 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.unordered_write: 0 2024-09-21T10:41:58.879 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-21T10:41:58.879 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-21T10:41:58.879 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-21T10:41:58.879 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-21T10:41:58.879 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.row_cache: None 2024-09-21T10:41:58.879 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.wal_filter: None 2024-09-21T10:41:58.879 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-21T10:41:58.879 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.allow_ingest_behind: 0 2024-09-21T10:41:58.879 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.two_write_queues: 0 2024-09-21T10:41:58.880 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.manual_wal_flush: 0 2024-09-21T10:41:58.880 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.wal_compression: 0 2024-09-21T10:41:58.880 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.atomic_flush: 0 2024-09-21T10:41:58.880 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-21T10:41:58.880 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-21T10:41:58.880 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-21T10:41:58.880 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.log_readahead_size: 0 2024-09-21T10:41:58.880 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-21T10:41:58.880 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.best_efforts_recovery: 0 2024-09-21T10:41:58.880 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-21T10:41:58.880 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-21T10:41:58.880 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.allow_data_in_errors: 0 2024-09-21T10:41:58.880 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.db_host_id: __hostname__ 2024-09-21T10:41:58.880 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-21T10:41:58.880 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.max_background_jobs: 2 2024-09-21T10:41:58.881 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.max_background_compactions: -1 2024-09-21T10:41:58.881 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.max_subcompactions: 1 2024-09-21T10:41:58.881 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-21T10:41:58.881 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-21T10:41:58.881 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-21T10:41:58.881 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.max_total_wal_size: 0 2024-09-21T10:41:58.881 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-21T10:41:58.881 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-21T10:41:58.881 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-21T10:41:58.881 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-21T10:41:58.881 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.max_open_files: -1 2024-09-21T10:41:58.881 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.bytes_per_sync: 0 2024-09-21T10:41:58.881 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-21T10:41:58.881 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-21T10:41:58.881 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.compaction_readahead_size: 0 2024-09-21T10:41:58.881 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.max_background_flushes: -1 2024-09-21T10:41:58.882 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Compression algorithms supported: 2024-09-21T10:41:58.882 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: kZSTD supported: 0 2024-09-21T10:41:58.882 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: kXpressCompression supported: 0 2024-09-21T10:41:58.882 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: kBZip2Compression supported: 0 2024-09-21T10:41:58.882 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-21T10:41:58.882 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: kLZ4Compression supported: 1 2024-09-21T10:41:58.882 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: kZlibCompression supported: 1 2024-09-21T10:41:58.882 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: kLZ4HCCompression supported: 1 2024-09-21T10:41:58.882 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: kSnappyCompression supported: 1 2024-09-21T10:41:58.882 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-21T10:41:58.882 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-21T10:41:58.882 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi175/store.db/MANIFEST-000009 2024-09-21T10:41:58.882 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-21T10:41:58.882 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-21T10:41:58.883 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.merge_operator: 2024-09-21T10:41:58.883 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.compaction_filter: None 2024-09-21T10:41:58.883 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.compaction_filter_factory: None 2024-09-21T10:41:58.883 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.sst_partitioner_factory: None 2024-09-21T10:41:58.883 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-21T10:41:58.883 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-21T10:41:58.883 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x563b26d53540) 2024-09-21T10:41:58.883 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: cache_index_and_filter_blocks: 1 2024-09-21T10:41:58.883 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-21T10:41:58.883 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-21T10:41:58.883 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: pin_top_level_index_and_filter: 1 2024-09-21T10:41:58.883 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: index_type: 0 2024-09-21T10:41:58.883 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: data_block_index_type: 0 2024-09-21T10:41:58.883 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: index_shortening: 1 2024-09-21T10:41:58.884 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-21T10:41:58.884 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: checksum: 4 2024-09-21T10:41:58.884 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: no_block_cache: 0 2024-09-21T10:41:58.884 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: block_cache: 0x563b26d47610 2024-09-21T10:41:58.884 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: block_cache_name: BinnedLRUCache 2024-09-21T10:41:58.884 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: block_cache_options: 2024-09-21T10:41:58.884 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: capacity : 536870912 2024-09-21T10:41:58.884 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: num_shard_bits : 4 2024-09-21T10:41:58.884 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: strict_capacity_limit : 0 2024-09-21T10:41:58.884 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: high_pri_pool_ratio: 0.000 2024-09-21T10:41:58.884 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: block_cache_compressed: (nil) 2024-09-21T10:41:58.884 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: persistent_cache: (nil) 2024-09-21T10:41:58.884 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: block_size: 4096 2024-09-21T10:41:58.884 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: block_size_deviation: 10 2024-09-21T10:41:58.885 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: block_restart_interval: 16 2024-09-21T10:41:58.885 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: index_block_restart_interval: 1 2024-09-21T10:41:58.885 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: metadata_block_size: 4096 2024-09-21T10:41:58.885 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: partition_filters: 0 2024-09-21T10:41:58.885 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: use_delta_encoding: 1 2024-09-21T10:41:58.885 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: filter_policy: bloomfilter 2024-09-21T10:41:58.885 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: whole_key_filtering: 1 2024-09-21T10:41:58.885 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: verify_compression: 0 2024-09-21T10:41:58.885 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: read_amp_bytes_per_bit: 0 2024-09-21T10:41:58.885 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: format_version: 5 2024-09-21T10:41:58.885 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: enable_index_compression: 1 2024-09-21T10:41:58.885 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: block_align: 0 2024-09-21T10:41:58.885 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: max_auto_readahead_size: 262144 2024-09-21T10:41:58.885 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: prepopulate_block_cache: 0 2024-09-21T10:41:58.885 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: initial_auto_readahead_size: 8192 2024-09-21T10:41:58.885 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: num_file_reads_for_auto_readahead: 2 2024-09-21T10:41:58.886 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.write_buffer_size: 33554432 2024-09-21T10:41:58.886 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.max_write_buffer_number: 2 2024-09-21T10:41:58.886 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.compression: NoCompression 2024-09-21T10:41:58.886 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.bottommost_compression: Disabled 2024-09-21T10:41:58.886 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.prefix_extractor: nullptr 2024-09-21T10:41:58.886 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-21T10:41:58.886 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.num_levels: 7 2024-09-21T10:41:58.886 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-21T10:41:58.886 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-21T10:41:58.886 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-21T10:41:58.886 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-21T10:41:58.886 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-21T10:41:58.886 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-21T10:41:58.886 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-21T10:41:58.887 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-21T10:41:58.887 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-21T10:41:58.887 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-21T10:41:58.887 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-21T10:41:58.887 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-21T10:41:58.887 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-21T10:41:58.887 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.compression_opts.level: 32767 2024-09-21T10:41:58.887 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.compression_opts.strategy: 0 2024-09-21T10:41:58.887 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-21T10:41:58.887 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-21T10:41:58.887 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-21T10:41:58.887 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-21T10:41:58.887 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.compression_opts.enabled: false 2024-09-21T10:41:58.887 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-21T10:41:58.888 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-21T10:41:58.888 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-21T10:41:58.888 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-21T10:41:58.888 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.target_file_size_base: 67108864 2024-09-21T10:41:58.888 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-21T10:41:58.888 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-21T10:41:58.888 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-21T10:41:58.888 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-21T10:41:58.888 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-21T10:41:58.888 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-21T10:41:58.888 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-21T10:41:58.888 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-21T10:41:58.888 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-21T10:41:58.888 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-21T10:41:58.888 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-21T10:41:58.889 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-21T10:41:58.889 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-21T10:41:58.889 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-21T10:41:58.889 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.arena_block_size: 1048576 2024-09-21T10:41:58.889 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-21T10:41:58.889 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-21T10:41:58.889 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.disable_auto_compactions: 0 2024-09-21T10:41:58.889 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-21T10:41:58.889 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-21T10:41:58.889 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-21T10:41:58.889 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-21T10:41:58.889 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-21T10:41:58.889 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-21T10:41:58.889 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-21T10:41:58.890 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-21T10:41:58.890 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-21T10:41:58.890 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-21T10:41:58.890 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-21T10:41:58.890 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.inplace_update_support: 0 2024-09-21T10:41:58.890 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-21T10:41:58.890 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-21T10:41:58.890 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-21T10:41:58.890 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-21T10:41:58.890 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.bloom_locality: 0 2024-09-21T10:41:58.890 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.max_successive_merges: 0 2024-09-21T10:41:58.890 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-21T10:41:58.890 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.paranoid_file_checks: 0 2024-09-21T10:41:58.890 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.force_consistency_checks: 1 2024-09-21T10:41:58.890 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.report_bg_io_stats: 0 2024-09-21T10:41:58.890 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.ttl: 2592000 2024-09-21T10:41:58.891 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-21T10:41:58.891 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-21T10:41:58.891 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-21T10:41:58.891 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.enable_blob_files: false 2024-09-21T10:41:58.891 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.min_blob_size: 0 2024-09-21T10:41:58.891 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.blob_file_size: 268435456 2024-09-21T10:41:58.891 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-21T10:41:58.891 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-21T10:41:58.891 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-21T10:41:58.891 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-21T10:41:58.891 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-21T10:41:58.891 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.blob_file_starting_level: 0 2024-09-21T10:41:58.891 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-21T10:41:58.891 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 24.sst 2024-09-21T10:41:58.891 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-21T10:41:58.892 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi175/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 26, last_sequence is 12254, log_number is 22,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-21T10:41:58.892 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 22 2024-09-21T10:41:58.892 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 79a04fb0-1f14-4258-8c3d-caac5d1d1b1e 2024-09-21T10:41:58.892 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726915318573736, "job": 1, "event": "recovery_started", "wal_files": [22]} 2024-09-21T10:41:58.892 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #22 mode 2 2024-09-21T10:41:58.892 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726915318592310, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 27, "file_size": 4156208, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12259, "largest_seqno": 13874, "table_properties": {"data_size": 4148866, "index_size": 4633, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1669, "raw_key_size": 15257, "raw_average_key_size": 23, "raw_value_size": 4134634, "raw_average_value_size": 6312, "num_data_blocks": 210, "num_entries": 655, "num_filter_entries": 655, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726915318, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "79a04fb0-1f14-4258-8c3d-caac5d1d1b1e", "db_session_id": "PNTTSECBWNBA16F57BOO", "orig_file_number": 27, "seqno_to_time_mapping": "N/A"}} 2024-09-21T10:41:58.892 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726915318592450, "job": 1, "event": "recovery_finished"} 2024-09-21T10:41:58.892 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: [db/version_set.cc:5047] Creating manifest 29 2024-09-21T10:41:58.892 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-21T10:41:58.892 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-smithi175/store.db/000022.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-21T10:41:58.892 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x563b26d76e00 2024-09-21T10:41:58.892 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: DB pointer 0x563b26e78000 2024-09-21T10:41:58.892 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-21T10:41:58.892 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-21T10:41:58.892 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: ** DB Stats ** 2024-09-21T10:41:58.893 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T10:41:58.893 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-21T10:41:58.893 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-21T10:41:58.893 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T10:41:58.893 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-21T10:41:58.893 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-21T10:41:58.893 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T10:41:58.893 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: 2024-09-21T10:41:58.893 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: ** Compaction Stats [default] ** 2024-09-21T10:41:58.893 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-21T10:41:58.893 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-21T10:41:58.893 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: L0 1/0 3.96 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 497.7 0.01 0.00 1 0.008 0 0 0.0 0.0 2024-09-21T10:41:58.893 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: L6 1/0 6.69 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-21T10:41:58.893 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: Sum 2/0 10.65 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 497.7 0.01 0.00 1 0.008 0 0 0.0 0.0 2024-09-21T10:41:58.893 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 497.7 0.01 0.00 1 0.008 0 0 0.0 0.0 2024-09-21T10:41:58.894 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: 2024-09-21T10:41:58.894 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: ** Compaction Stats [default] ** 2024-09-21T10:41:58.894 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-21T10:41:58.894 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-21T10:41:58.894 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 497.7 0.01 0.00 1 0.008 0 0 0.0 0.0 2024-09-21T10:41:58.894 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: 2024-09-21T10:41:58.894 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-21T10:41:58.894 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: 2024-09-21T10:41:58.894 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T10:41:58.894 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: Flush(GB): cumulative 0.004, interval 0.004 2024-09-21T10:41:58.894 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-21T10:41:58.894 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-21T10:41:58.894 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-21T10:41:58.894 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-21T10:41:58.895 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: Cumulative compaction: 0.00 GB write, 160.28 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T10:41:58.895 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: Interval compaction: 0.00 GB write, 160.28 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T10:41:58.895 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-21T10:41:58.895 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: Block cache BinnedLRUCache@0x563b26d47610#2 capacity: 512.00 MB usage: 6.86 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.2e-05 secs_since: 0 2024-09-21T10:41:58.895 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.78 KB,0.000339746%) IndexBlock(1,5.08 KB,0.000968575%) Misc(1,0.00 KB,0%) 2024-09-21T10:41:58.895 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: 2024-09-21T10:41:58.895 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-21T10:41:58.895 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: starting mon.smithi175 rank 1 at public addrs [v2:172.21.15.175:3300/0,v1:172.21.15.175:6789/0] at bind addrs [v2:172.21.15.175:3300/0,v1:172.21.15.175:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi175 fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 2024-09-21T10:41:58.895 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: mon.smithi175@-1(???) e2 preinit fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 2024-09-21T10:41:58.895 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: mon.smithi175@-1(???).mds e6 new map 2024-09-21T10:41:58.895 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: mon.smithi175@-1(???).mds e6 print_map 2024-09-21T10:41:58.895 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: e6 2024-09-21T10:41:58.895 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-21T10:41:58.895 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-21T10:41:58.895 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-21T10:41:58.896 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: legacy client fscid: 1 2024-09-21T10:41:58.896 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: 2024-09-21T10:41:58.896 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: Filesystem 'foofs' (1) 2024-09-21T10:41:58.896 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: fs_name foofs 2024-09-21T10:41:58.896 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: epoch 4 2024-09-21T10:41:58.896 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-21T10:41:58.896 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: created 2024-09-21T10:36:25.765834+0000 2024-09-21T10:41:58.896 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: modified 2024-09-21T10:36:31.639311+0000 2024-09-21T10:41:58.896 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: tableserver 0 2024-09-21T10:41:58.896 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: root 0 2024-09-21T10:41:58.896 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: session_timeout 60 2024-09-21T10:41:58.896 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: session_autoclose 300 2024-09-21T10:41:58.896 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: max_file_size 1099511627776 2024-09-21T10:41:58.896 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: max_xattr_size 65536 2024-09-21T10:41:58.896 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: required_client_features {} 2024-09-21T10:41:58.897 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: last_failure 0 2024-09-21T10:41:58.897 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: last_failure_osd_epoch 0 2024-09-21T10:41:58.897 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-21T10:41:58.897 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: max_mds 1 2024-09-21T10:41:58.897 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: in 0 2024-09-21T10:41:58.897 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: up {0=14492} 2024-09-21T10:41:58.897 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: failed 2024-09-21T10:41:58.897 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: damaged 2024-09-21T10:41:58.897 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: stopped 2024-09-21T10:41:58.897 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: data_pools [3] 2024-09-21T10:41:58.897 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: metadata_pool 2 2024-09-21T10:41:58.897 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: inline_data disabled 2024-09-21T10:41:58.897 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: balancer 2024-09-21T10:41:58.898 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: bal_rank_mask -1 2024-09-21T10:41:58.898 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: standby_count_wanted 1 2024-09-21T10:41:58.898 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: qdb_cluster leader: 0 members: 2024-09-21T10:41:58.898 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: [mds.foofs.smithi123.uqoyow{0:14492} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.123:6834/625956046,v1:172.21.15.123:6835/625956046] compat {c=[1],r=[1],i=[7ff]}] 2024-09-21T10:41:58.898 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: 2024-09-21T10:41:58.898 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: 2024-09-21T10:41:58.898 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: Standby daemons: 2024-09-21T10:41:58.898 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: 2024-09-21T10:41:58.898 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout: [mds.foofs.smithi175.jrlevw{-1:24269} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.175:6832/3454320220,v1:172.21.15.175:6833/3454320220] compat {c=[1],r=[1],i=[7ff]}] 2024-09-21T10:41:58.898 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: mon.smithi175@-1(???).osd e76 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-21T10:41:58.898 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: mon.smithi175@-1(???).osd e76 crush map has features 432629239337189376, adjusting msgr requires 2024-09-21T10:41:58.898 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: mon.smithi175@-1(???).osd e76 crush map has features 432629239337189376, adjusting msgr requires 2024-09-21T10:41:58.898 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: mon.smithi175@-1(???).osd e76 crush map has features 432629239337189376, adjusting msgr requires 2024-09-21T10:41:58.898 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 ceph-mon[49304]: mon.smithi175@-1(???).paxosservice(auth 1..30) refresh upgraded, format 0 -> 3 2024-09-21T10:41:58.898 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 bash[49285]: 3abee2d37f0f1c7e54993398d30da21d183d47b7a44609532666b16d2f74b204 2024-09-21T10:41:58.898 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:41:58 smithi175 systemd[1]: Started Ceph mon.smithi175 for 3ccd15e6-7804-11ef-baf6-efdc52797490. 2024-09-21T10:41:59.817 INFO:teuthology.orchestra.run.smithi123.stdout: 5 129193 10.26 MB/sec execute 180 sec latency 76.751 ms 2024-09-21T10:42:00.817 INFO:teuthology.orchestra.run.smithi123.stdout: 5 129402 10.24 MB/sec execute 181 sec latency 57.489 ms 2024-09-21T10:42:01.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:01 smithi175 ceph-mon[49304]: fsmap foofs:1 {0=foofs.smithi123.uqoyow=up:active} 1 up:standby 2024-09-21T10:42:01.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:01 smithi175 ceph-mon[49304]: osdmap e76: 8 total, 8 up, 8 in 2024-09-21T10:42:01.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:01 smithi175 ceph-mon[49304]: mgrmap e40: smithi123.stnuit(active, since 18s), standbys: smithi175.txvwrq 2024-09-21T10:42:01.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:01 smithi175 ceph-mon[49304]: overall HEALTH_OK 2024-09-21T10:42:01.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:01 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "mon metadata", "id": "smithi123"} : dispatch 2024-09-21T10:42:01.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:01 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "mon metadata", "id": "smithi175"} : dispatch 2024-09-21T10:42:01.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:01 smithi175 ceph-mon[49304]: mon.smithi175 calling monitor election 2024-09-21T10:42:01.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:01 smithi175 ceph-mon[49304]: mon.smithi123 calling monitor election 2024-09-21T10:42:01.375 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:01 smithi175 ceph-mon[49304]: mon.smithi123 is new leader, mons smithi123,smithi175 in quorum (ranks 0,1) 2024-09-21T10:42:01.375 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:01 smithi175 ceph-mon[49304]: monmap epoch 3 2024-09-21T10:42:01.375 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:01 smithi175 ceph-mon[49304]: fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 2024-09-21T10:42:01.375 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:01 smithi175 ceph-mon[49304]: last_changed 2024-09-21T10:42:00.179237+0000 2024-09-21T10:42:01.375 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:01 smithi175 ceph-mon[49304]: created 2024-09-21T10:28:48.510560+0000 2024-09-21T10:42:01.375 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:01 smithi175 ceph-mon[49304]: min_mon_release 19 (squid) 2024-09-21T10:42:01.375 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:01 smithi175 ceph-mon[49304]: election_strategy: 1 2024-09-21T10:42:01.375 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:01 smithi175 ceph-mon[49304]: 0: [v2:172.21.15.123:3300/0,v1:172.21.15.123:6789/0] mon.smithi123 2024-09-21T10:42:01.375 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:01 smithi175 ceph-mon[49304]: 1: [v2:172.21.15.175:3300/0,v1:172.21.15.175:6789/0] mon.smithi175 2024-09-21T10:42:01.375 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:01 smithi175 ceph-mon[49304]: fsmap foofs:1 {0=foofs.smithi123.uqoyow=up:active} 1 up:standby 2024-09-21T10:42:01.375 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:01 smithi175 ceph-mon[49304]: osdmap e76: 8 total, 8 up, 8 in 2024-09-21T10:42:01.375 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:01 smithi175 ceph-mon[49304]: mgrmap e40: smithi123.stnuit(active, since 18s), standbys: smithi175.txvwrq 2024-09-21T10:42:01.375 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:01 smithi175 ceph-mon[49304]: overall HEALTH_OK 2024-09-21T10:42:01.375 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:01 smithi175 ceph-mon[49304]: pgmap v12: 97 pgs: 97 active+clean; 342 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 24 KiB/s rd, 5.4 MiB/s wr, 552 op/s 2024-09-21T10:42:01.375 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:01 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:01.376 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:01 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:01.376 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:01 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:42:01.385 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:01 smithi123 ceph-mon[63856]: fsmap foofs:1 {0=foofs.smithi123.uqoyow=up:active} 1 up:standby 2024-09-21T10:42:01.386 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:01 smithi123 ceph-mon[63856]: osdmap e76: 8 total, 8 up, 8 in 2024-09-21T10:42:01.386 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:01 smithi123 ceph-mon[63856]: mgrmap e40: smithi123.stnuit(active, since 18s), standbys: smithi175.txvwrq 2024-09-21T10:42:01.386 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:01 smithi123 ceph-mon[63856]: overall HEALTH_OK 2024-09-21T10:42:01.386 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:01 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "mon metadata", "id": "smithi123"} : dispatch 2024-09-21T10:42:01.386 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:01 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "mon metadata", "id": "smithi175"} : dispatch 2024-09-21T10:42:01.386 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:01 smithi123 ceph-mon[63856]: mon.smithi175 calling monitor election 2024-09-21T10:42:01.386 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:01 smithi123 ceph-mon[63856]: mon.smithi123 calling monitor election 2024-09-21T10:42:01.386 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:01 smithi123 ceph-mon[63856]: mon.smithi123 is new leader, mons smithi123,smithi175 in quorum (ranks 0,1) 2024-09-21T10:42:01.386 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:01 smithi123 ceph-mon[63856]: monmap epoch 3 2024-09-21T10:42:01.386 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:01 smithi123 ceph-mon[63856]: fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 2024-09-21T10:42:01.386 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:01 smithi123 ceph-mon[63856]: last_changed 2024-09-21T10:42:00.179237+0000 2024-09-21T10:42:01.386 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:01 smithi123 ceph-mon[63856]: created 2024-09-21T10:28:48.510560+0000 2024-09-21T10:42:01.386 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:01 smithi123 ceph-mon[63856]: min_mon_release 19 (squid) 2024-09-21T10:42:01.386 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:01 smithi123 ceph-mon[63856]: election_strategy: 1 2024-09-21T10:42:01.386 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:01 smithi123 ceph-mon[63856]: 0: [v2:172.21.15.123:3300/0,v1:172.21.15.123:6789/0] mon.smithi123 2024-09-21T10:42:01.387 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:01 smithi123 ceph-mon[63856]: 1: [v2:172.21.15.175:3300/0,v1:172.21.15.175:6789/0] mon.smithi175 2024-09-21T10:42:01.387 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:01 smithi123 ceph-mon[63856]: fsmap foofs:1 {0=foofs.smithi123.uqoyow=up:active} 1 up:standby 2024-09-21T10:42:01.387 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:01 smithi123 ceph-mon[63856]: osdmap e76: 8 total, 8 up, 8 in 2024-09-21T10:42:01.387 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:01 smithi123 ceph-mon[63856]: mgrmap e40: smithi123.stnuit(active, since 18s), standbys: smithi175.txvwrq 2024-09-21T10:42:01.387 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:01 smithi123 ceph-mon[63856]: overall HEALTH_OK 2024-09-21T10:42:01.387 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:01 smithi123 ceph-mon[63856]: pgmap v12: 97 pgs: 97 active+clean; 342 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 24 KiB/s rd, 5.4 MiB/s wr, 552 op/s 2024-09-21T10:42:01.387 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:01 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:01.387 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:01 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:01.387 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:01 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:42:01.817 INFO:teuthology.orchestra.run.smithi123.stdout: 5 129904 10.28 MB/sec execute 182 sec latency 60.232 ms 2024-09-21T10:42:02.817 INFO:teuthology.orchestra.run.smithi123.stdout: 5 130227 10.25 MB/sec execute 183 sec latency 60.240 ms 2024-09-21T10:42:03.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:03 smithi175 ceph-mon[49304]: pgmap v13: 97 pgs: 97 active+clean; 342 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.9 MiB/s wr, 371 op/s 2024-09-21T10:42:03.443 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:03 smithi123 ceph-mon[63856]: pgmap v13: 97 pgs: 97 active+clean; 342 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.9 MiB/s wr, 371 op/s 2024-09-21T10:42:03.817 INFO:teuthology.orchestra.run.smithi123.stdout: 5 130727 10.29 MB/sec execute 184 sec latency 67.106 ms 2024-09-21T10:42:04.817 INFO:teuthology.orchestra.run.smithi123.stdout: 5 130963 10.26 MB/sec execute 185 sec latency 54.016 ms 2024-09-21T10:42:05.473 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:05 smithi123 ceph-mon[63856]: pgmap v14: 97 pgs: 97 active+clean; 340 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.6 MiB/s wr, 575 op/s 2024-09-21T10:42:05.473 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:05 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:05.473 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:05 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:05.473 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:05 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:05.474 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:05 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:05.608 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:05 smithi175 ceph-mon[49304]: pgmap v14: 97 pgs: 97 active+clean; 340 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.6 MiB/s wr, 575 op/s 2024-09-21T10:42:05.608 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:05 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:05.608 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:05 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:05.608 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:05 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:05.608 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:05 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:05.817 INFO:teuthology.orchestra.run.smithi123.stdout: 5 131229 10.28 MB/sec execute 186 sec latency 60.206 ms 2024-09-21T10:42:06.817 INFO:teuthology.orchestra.run.smithi123.stdout: 5 131495 10.25 MB/sec execute 187 sec latency 69.208 ms 2024-09-21T10:42:07.543 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:07 smithi123 ceph-mon[63856]: pgmap v15: 97 pgs: 97 active+clean; 340 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 3.3 MiB/s wr, 352 op/s 2024-09-21T10:42:07.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:07 smithi175 ceph-mon[49304]: pgmap v15: 97 pgs: 97 active+clean; 340 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 3.3 MiB/s wr, 352 op/s 2024-09-21T10:42:07.817 INFO:teuthology.orchestra.run.smithi123.stdout: 5 131710 10.24 MB/sec execute 188 sec latency 72.134 ms 2024-09-21T10:42:08.817 INFO:teuthology.orchestra.run.smithi123.stdout: 5 131909 10.21 MB/sec execute 189 sec latency 114.183 ms 2024-09-21T10:42:09.817 INFO:teuthology.orchestra.run.smithi123.stdout: 5 132192 10.20 MB/sec execute 190 sec latency 99.663 ms 2024-09-21T10:42:09.838 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:09 smithi123 ceph-mon[63856]: pgmap v16: 97 pgs: 97 active+clean; 340 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 3.3 MiB/s wr, 352 op/s 2024-09-21T10:42:09.838 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:09 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:09.838 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:09 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:09.838 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:09 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:42:09.838 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:09 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:42:09.839 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:09 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:09.839 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:09 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:09.839 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:09 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T10:42:09.839 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:09 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T10:42:09.839 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:09 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:42:10.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:09 smithi175 ceph-mon[49304]: pgmap v16: 97 pgs: 97 active+clean; 340 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 3.3 MiB/s wr, 352 op/s 2024-09-21T10:42:10.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:09 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:10.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:09 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:10.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:09 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:42:10.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:09 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:42:10.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:09 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:10.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:09 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:10.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:09 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T10:42:10.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:09 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T10:42:10.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:09 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:42:10.769 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:10 smithi123 ceph-mon[63856]: Reconfiguring mon.smithi123 (monmap changed)... 2024-09-21T10:42:10.769 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:10 smithi123 ceph-mon[63856]: Reconfiguring daemon mon.smithi123 on smithi123 2024-09-21T10:42:10.817 INFO:teuthology.orchestra.run.smithi123.stdout: 5 132532 10.16 MB/sec execute 191 sec latency 59.943 ms 2024-09-21T10:42:11.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:10 smithi175 ceph-mon[49304]: Reconfiguring mon.smithi123 (monmap changed)... 2024-09-21T10:42:11.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:10 smithi175 ceph-mon[49304]: Reconfiguring daemon mon.smithi123 on smithi123 2024-09-21T10:42:11.817 INFO:teuthology.orchestra.run.smithi123.stdout: 5 132809 10.12 MB/sec execute 192 sec latency 50.030 ms 2024-09-21T10:42:11.895 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:11 smithi123 ceph-mon[63856]: pgmap v17: 97 pgs: 97 active+clean; 343 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.3 MiB/s wr, 524 op/s 2024-09-21T10:42:11.895 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:11 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:11.895 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:11 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:11.895 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:11 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get-or-create", "entity": "mgr.smithi123.stnuit", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-21T10:42:11.895 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:11 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "mgr services"} : dispatch 2024-09-21T10:42:11.895 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:11 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:42:12.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:11 smithi175 ceph-mon[49304]: pgmap v17: 97 pgs: 97 active+clean; 343 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.3 MiB/s wr, 524 op/s 2024-09-21T10:42:12.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:11 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:12.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:11 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:12.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:11 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get-or-create", "entity": "mgr.smithi123.stnuit", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-21T10:42:12.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:11 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "mgr services"} : dispatch 2024-09-21T10:42:12.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:11 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:42:12.817 INFO:teuthology.orchestra.run.smithi123.stdout: 5 133011 10.11 MB/sec execute 193 sec latency 62.806 ms 2024-09-21T10:42:12.927 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:12 smithi123 ceph-mon[63856]: Reconfiguring mgr.smithi123.stnuit (monmap changed)... 2024-09-21T10:42:12.927 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:12 smithi123 ceph-mon[63856]: Reconfiguring daemon mgr.smithi123.stnuit on smithi123 2024-09-21T10:42:12.927 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:12 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:42:13.024 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:12 smithi175 ceph-mon[49304]: Reconfiguring mgr.smithi123.stnuit (monmap changed)... 2024-09-21T10:42:13.024 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:12 smithi175 ceph-mon[49304]: Reconfiguring daemon mgr.smithi123.stnuit on smithi123 2024-09-21T10:42:13.024 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:12 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:42:13.702 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:13 smithi123 ceph-mon[63856]: pgmap v18: 97 pgs: 97 active+clean; 343 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.8 MiB/s wr, 375 op/s 2024-09-21T10:42:13.702 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:13 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:13.702 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:13 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:13.702 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:13 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi123", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-21T10:42:13.702 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:13 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:42:13.817 INFO:teuthology.orchestra.run.smithi123.stdout: 5 133319 10.10 MB/sec execute 194 sec latency 64.606 ms 2024-09-21T10:42:13.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:13 smithi175 ceph-mon[49304]: pgmap v18: 97 pgs: 97 active+clean; 343 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.8 MiB/s wr, 375 op/s 2024-09-21T10:42:13.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:13 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:13.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:13 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:13.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:13 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi123", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-21T10:42:13.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:13 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:42:14.726 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:14 smithi123 ceph-mon[63856]: Reconfiguring crash.smithi123 (monmap changed)... 2024-09-21T10:42:14.726 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:14 smithi123 ceph-mon[63856]: Reconfiguring daemon crash.smithi123 on smithi123 2024-09-21T10:42:14.817 INFO:teuthology.orchestra.run.smithi123.stdout: 5 133649 10.10 MB/sec execute 195 sec latency 45.794 ms 2024-09-21T10:42:15.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:14 smithi175 ceph-mon[49304]: Reconfiguring crash.smithi123 (monmap changed)... 2024-09-21T10:42:15.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:14 smithi175 ceph-mon[49304]: Reconfiguring daemon crash.smithi123 on smithi123 2024-09-21T10:42:15.818 INFO:teuthology.orchestra.run.smithi123.stdout: 5 134121 10.10 MB/sec execute 196 sec latency 34.653 ms 2024-09-21T10:42:15.893 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:15 smithi123 ceph-mon[63856]: pgmap v19: 97 pgs: 97 active+clean; 335 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 5.3 MiB/s wr, 556 op/s 2024-09-21T10:42:15.893 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:15 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:15.893 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:15 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:15.893 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:15 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-21T10:42:15.893 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:15 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:42:16.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:15 smithi175 ceph-mon[49304]: pgmap v19: 97 pgs: 97 active+clean; 335 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 5.3 MiB/s wr, 556 op/s 2024-09-21T10:42:16.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:15 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:16.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:15 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:16.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:15 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-21T10:42:16.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:15 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:42:16.818 INFO:teuthology.orchestra.run.smithi123.stdout: 5 134441 10.12 MB/sec execute 197 sec latency 34.568 ms 2024-09-21T10:42:16.903 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:16 smithi123 ceph-mon[63856]: Reconfiguring osd.0 (monmap changed)... 2024-09-21T10:42:16.903 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:16 smithi123 ceph-mon[63856]: Reconfiguring daemon osd.0 on smithi123 2024-09-21T10:42:17.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:16 smithi175 ceph-mon[49304]: Reconfiguring osd.0 (monmap changed)... 2024-09-21T10:42:17.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:16 smithi175 ceph-mon[49304]: Reconfiguring daemon osd.0 on smithi123 2024-09-21T10:42:17.818 INFO:teuthology.orchestra.run.smithi123.stdout: 5 134804 10.14 MB/sec execute 198 sec latency 63.031 ms 2024-09-21T10:42:17.924 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:17 smithi123 ceph-mon[63856]: pgmap v20: 97 pgs: 97 active+clean; 335 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.6 MiB/s wr, 352 op/s 2024-09-21T10:42:17.924 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:17 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:17.924 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:17 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:17.924 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:17 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-21T10:42:17.924 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:17 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:42:18.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:17 smithi175 ceph-mon[49304]: pgmap v20: 97 pgs: 97 active+clean; 335 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.6 MiB/s wr, 352 op/s 2024-09-21T10:42:18.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:17 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:18.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:17 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:18.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:17 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-21T10:42:18.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:17 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:42:18.818 INFO:teuthology.orchestra.run.smithi123.stdout: 5 135146 10.14 MB/sec execute 199 sec latency 43.772 ms 2024-09-21T10:42:18.937 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:18 smithi123 ceph-mon[63856]: Reconfiguring osd.1 (monmap changed)... 2024-09-21T10:42:18.937 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:18 smithi123 ceph-mon[63856]: Reconfiguring daemon osd.1 on smithi123 2024-09-21T10:42:19.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:18 smithi175 ceph-mon[49304]: Reconfiguring osd.1 (monmap changed)... 2024-09-21T10:42:19.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:18 smithi175 ceph-mon[49304]: Reconfiguring daemon osd.1 on smithi123 2024-09-21T10:42:19.818 INFO:teuthology.orchestra.run.smithi123.stdout: 5 135698 10.15 MB/sec execute 200 sec latency 32.885 ms 2024-09-21T10:42:19.945 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:19 smithi123 ceph-mon[63856]: pgmap v21: 97 pgs: 97 active+clean; 335 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.6 MiB/s wr, 352 op/s 2024-09-21T10:42:20.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:19 smithi175 ceph-mon[49304]: pgmap v21: 97 pgs: 97 active+clean; 335 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.6 MiB/s wr, 352 op/s 2024-09-21T10:42:20.818 INFO:teuthology.orchestra.run.smithi123.stdout: 5 136338 10.14 MB/sec execute 201 sec latency 39.792 ms 2024-09-21T10:42:21.200 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:21 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:21.200 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:21 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:21.200 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:21 smithi123 ceph-mon[63856]: Reconfiguring osd.2 (monmap changed)... 2024-09-21T10:42:21.200 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:21 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-21T10:42:21.200 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:21 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:42:21.200 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:21 smithi123 ceph-mon[63856]: Reconfiguring daemon osd.2 on smithi123 2024-09-21T10:42:21.201 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:21 smithi123 ceph-mon[63856]: pgmap v22: 97 pgs: 97 active+clean; 347 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.9 MiB/s wr, 555 op/s 2024-09-21T10:42:21.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:21 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:21.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:21 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:21.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:21 smithi175 ceph-mon[49304]: Reconfiguring osd.2 (monmap changed)... 2024-09-21T10:42:21.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:21 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-21T10:42:21.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:21 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:42:21.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:21 smithi175 ceph-mon[49304]: Reconfiguring daemon osd.2 on smithi123 2024-09-21T10:42:21.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:21 smithi175 ceph-mon[49304]: pgmap v22: 97 pgs: 97 active+clean; 347 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.9 MiB/s wr, 555 op/s 2024-09-21T10:42:21.818 INFO:teuthology.orchestra.run.smithi123.stdout: 5 137217 10.23 MB/sec execute 202 sec latency 21.064 ms 2024-09-21T10:42:22.818 INFO:teuthology.orchestra.run.smithi123.stdout: 5 137894 10.28 MB/sec execute 203 sec latency 24.267 ms 2024-09-21T10:42:23.715 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:23 smithi123 ceph-mon[63856]: pgmap v23: 97 pgs: 97 active+clean; 347 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.9 MiB/s wr, 383 op/s 2024-09-21T10:42:23.715 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:23 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:23.715 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:23 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:23.715 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:23 smithi123 ceph-mon[63856]: Reconfiguring osd.3 (monmap changed)... 2024-09-21T10:42:23.715 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:23 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-21T10:42:23.716 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:23 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:42:23.716 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:23 smithi123 ceph-mon[63856]: Reconfiguring daemon osd.3 on smithi123 2024-09-21T10:42:23.818 INFO:teuthology.orchestra.run.smithi123.stdout: 5 138582 10.34 MB/sec execute 204 sec latency 25.859 ms 2024-09-21T10:42:23.833 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:23 smithi175 ceph-mon[49304]: pgmap v23: 97 pgs: 97 active+clean; 347 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.9 MiB/s wr, 383 op/s 2024-09-21T10:42:23.833 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:23 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:23.833 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:23 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:23.833 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:23 smithi175 ceph-mon[49304]: Reconfiguring osd.3 (monmap changed)... 2024-09-21T10:42:23.833 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:23 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-21T10:42:23.833 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:23 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:42:23.833 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:23 smithi175 ceph-mon[49304]: Reconfiguring daemon osd.3 on smithi123 2024-09-21T10:42:24.818 INFO:teuthology.orchestra.run.smithi123.stdout: 5 139184 10.37 MB/sec execute 205 sec latency 48.786 ms 2024-09-21T10:42:25.731 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:25 smithi123 ceph-mon[63856]: pgmap v24: 97 pgs: 97 active+clean; 348 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 7.8 MiB/s wr, 769 op/s 2024-09-21T10:42:25.818 INFO:teuthology.orchestra.run.smithi123.stdout: 5 139869 10.36 MB/sec execute 206 sec latency 43.611 ms 2024-09-21T10:42:25.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:25 smithi175 ceph-mon[49304]: pgmap v24: 97 pgs: 97 active+clean; 348 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 7.8 MiB/s wr, 769 op/s 2024-09-21T10:42:26.739 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:26 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:26.739 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:26 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:26.740 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:26 smithi123 ceph-mon[63856]: Reconfiguring mds.foofs.smithi123.uqoyow (monmap changed)... 2024-09-21T10:42:26.740 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:26 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi123.uqoyow", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-21T10:42:26.740 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:26 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:42:26.740 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:26 smithi123 ceph-mon[63856]: Reconfiguring daemon mds.foofs.smithi123.uqoyow on smithi123 2024-09-21T10:42:26.818 INFO:teuthology.orchestra.run.smithi123.stdout: 5 140619 10.43 MB/sec execute 207 sec latency 40.254 ms 2024-09-21T10:42:27.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:26 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:27.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:26 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:27.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:26 smithi175 ceph-mon[49304]: Reconfiguring mds.foofs.smithi123.uqoyow (monmap changed)... 2024-09-21T10:42:27.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:26 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi123.uqoyow", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-21T10:42:27.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:26 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:42:27.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:26 smithi175 ceph-mon[49304]: Reconfiguring daemon mds.foofs.smithi123.uqoyow on smithi123 2024-09-21T10:42:27.744 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:27 smithi123 ceph-mon[63856]: pgmap v25: 97 pgs: 97 active+clean; 348 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 6.2 MiB/s wr, 588 op/s 2024-09-21T10:42:27.744 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:27 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:42:27.818 INFO:teuthology.orchestra.run.smithi123.stdout: 5 141312 10.47 MB/sec execute 208 sec latency 31.014 ms 2024-09-21T10:42:27.827 INFO:teuthology.orchestra.run.smithi123.stdout:true 2024-09-21T10:42:28.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:27 smithi175 ceph-mon[49304]: pgmap v25: 97 pgs: 97 active+clean; 348 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 6.2 MiB/s wr, 588 op/s 2024-09-21T10:42:28.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:27 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:42:28.818 INFO:teuthology.orchestra.run.smithi123.stdout: 5 141667 10.48 MB/sec execute 209 sec latency 37.512 ms 2024-09-21T10:42:28.900 INFO:teuthology.orchestra.run.smithi123.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T10:42:28.900 INFO:teuthology.orchestra.run.smithi123.stdout:alertmanager.smithi123 smithi123 *:9093,9094 running (2m) 37s ago 12m 19.7M - 0.27.0 11f11916f8cd 5d550bd97de5 2024-09-21T10:42:28.900 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi123 smithi123 running (12m) 37s ago 12m 6405k - 17.2.0 e1d6a67b021e c459f87d5b50 2024-09-21T10:42:28.901 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi175 smithi175 running (11m) 24s ago 11m 6405k - 17.2.0 e1d6a67b021e e35e422d5583 2024-09-21T10:42:28.901 INFO:teuthology.orchestra.run.smithi123.stdout:grafana.smithi123 smithi123 *:3000 running (2m) 37s ago 11m 43.4M - dad864ee21e9 771df81763ba 2024-09-21T10:42:28.901 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi123.uqoyow smithi123 running (5m) 37s ago 5m 40.4M - 17.2.0 e1d6a67b021e f0a21eaac266 2024-09-21T10:42:28.901 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi175.jrlevw smithi175 running (5m) 24s ago 5m 13.4M - 17.2.0 e1d6a67b021e b21b16f4bc50 2024-09-21T10:42:28.901 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi123.stnuit smithi123 *:8443,9283,8765 running (2m) 37s ago 13m 489M - 19.3.0-5103-g5c1cdc3c 8f4d013db793 6812cc5cb56b 2024-09-21T10:42:28.901 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi175.txvwrq smithi175 *:8443,9283,8765 running (70s) 24s ago 11m 453M - 19.3.0-5103-g5c1cdc3c 8f4d013db793 304f0408075e 2024-09-21T10:42:28.901 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi123 smithi123 running (54s) 37s ago 13m 30.6M 2048M 19.3.0-5103-g5c1cdc3c 8f4d013db793 6b95207a2d80 2024-09-21T10:42:28.901 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi175 smithi175 running (30s) 24s ago 11m 24.5M 2048M 19.3.0-5103-g5c1cdc3c 8f4d013db793 3abee2d37f0f 2024-09-21T10:42:28.901 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.0.0.smithi123.ddatpr smithi123 *:2049 running (5m) 37s ago 5m 275M - 4.0 e1d6a67b021e 36f8cb2281eb 2024-09-21T10:42:28.901 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.1.0.smithi175.wpawer smithi175 *:2049 running (5m) 24s ago 5m 90.2M - 4.0 e1d6a67b021e 2200aeed9200 2024-09-21T10:42:28.901 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi123 smithi123 *:9100 running (2m) 37s ago 11m 10.2M - 1.7.0 72c9c2088986 ca7d0b29af81 2024-09-21T10:42:28.901 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi175 smithi175 *:9100 running (2m) 24s ago 11m 11.1M - 1.7.0 72c9c2088986 46c4f6b7c149 2024-09-21T10:42:28.901 INFO:teuthology.orchestra.run.smithi123.stdout:osd.0 smithi123 running (10m) 37s ago 10m 449M 1936M 17.2.0 e1d6a67b021e a94edf88c456 2024-09-21T10:42:28.902 INFO:teuthology.orchestra.run.smithi123.stdout:osd.1 smithi123 running (10m) 37s ago 9m 404M 1936M 17.2.0 e1d6a67b021e 59aee515b0f8 2024-09-21T10:42:28.902 INFO:teuthology.orchestra.run.smithi123.stdout:osd.2 smithi123 running (9m) 37s ago 9m 293M 1936M 17.2.0 e1d6a67b021e 7db93f8ce5b5 2024-09-21T10:42:28.902 INFO:teuthology.orchestra.run.smithi123.stdout:osd.3 smithi123 running (9m) 37s ago 9m 420M 1936M 17.2.0 e1d6a67b021e c1672dea7b1f 2024-09-21T10:42:28.902 INFO:teuthology.orchestra.run.smithi123.stdout:osd.4 smithi175 running (8m) 24s ago 8m 455M 2704M 17.2.0 e1d6a67b021e 48fe728c259e 2024-09-21T10:42:28.902 INFO:teuthology.orchestra.run.smithi123.stdout:osd.5 smithi175 running (8m) 24s ago 8m 515M 2704M 17.2.0 e1d6a67b021e e270d5a21b14 2024-09-21T10:42:28.902 INFO:teuthology.orchestra.run.smithi123.stdout:osd.6 smithi175 running (7m) 24s ago 7m 435M 2704M 17.2.0 e1d6a67b021e 9bcbf018eb20 2024-09-21T10:42:28.902 INFO:teuthology.orchestra.run.smithi123.stdout:osd.7 smithi175 running (7m) 24s ago 7m 319M 2704M 17.2.0 e1d6a67b021e e9b8f1f90994 2024-09-21T10:42:28.902 INFO:teuthology.orchestra.run.smithi123.stdout:prometheus.smithi123 smithi123 *:9095 running (76s) 37s ago 10m 46.0M - 2.51.0 1d3b7f56885b 37f944c9de88 2024-09-21T10:42:29.288 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:29 smithi123 ceph-mon[63856]: from='client.34156 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:42:29.288 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:29 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:29.288 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:29 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:29.288 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:29 smithi123 ceph-mon[63856]: Reconfiguring crash.smithi175 (monmap changed)... 2024-09-21T10:42:29.288 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:29 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi175", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-21T10:42:29.288 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:29 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:42:29.288 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:29 smithi123 ceph-mon[63856]: Reconfiguring daemon crash.smithi175 on smithi175 2024-09-21T10:42:29.288 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:29 smithi123 ceph-mon[63856]: pgmap v26: 97 pgs: 97 active+clean; 348 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 6.2 MiB/s wr, 588 op/s 2024-09-21T10:42:29.289 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:29 smithi123 ceph-mon[63856]: from='client.44105 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:42:29.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:29 smithi175 ceph-mon[49304]: from='client.34156 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:42:29.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:29 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:29.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:29 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:29.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:29 smithi175 ceph-mon[49304]: Reconfiguring crash.smithi175 (monmap changed)... 2024-09-21T10:42:29.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:29 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi175", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-21T10:42:29.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:29 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:42:29.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:29 smithi175 ceph-mon[49304]: Reconfiguring daemon crash.smithi175 on smithi175 2024-09-21T10:42:29.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:29 smithi175 ceph-mon[49304]: pgmap v26: 97 pgs: 97 active+clean; 348 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 6.2 MiB/s wr, 588 op/s 2024-09-21T10:42:29.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:29 smithi175 ceph-mon[49304]: from='client.44105 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:42:29.519 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-09-21T10:42:29.519 INFO:teuthology.orchestra.run.smithi123.stdout: "mon": { 2024-09-21T10:42:29.519 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev)": 2 2024-09-21T10:42:29.519 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:42:29.519 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr": { 2024-09-21T10:42:29.519 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev)": 2 2024-09-21T10:42:29.519 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:42:29.519 INFO:teuthology.orchestra.run.smithi123.stdout: "osd": { 2024-09-21T10:42:29.519 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-21T10:42:29.519 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:42:29.520 INFO:teuthology.orchestra.run.smithi123.stdout: "mds": { 2024-09-21T10:42:29.520 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T10:42:29.520 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:42:29.520 INFO:teuthology.orchestra.run.smithi123.stdout: "overall": { 2024-09-21T10:42:29.520 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-09-21T10:42:29.520 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev)": 4 2024-09-21T10:42:29.520 INFO:teuthology.orchestra.run.smithi123.stdout: } 2024-09-21T10:42:29.520 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-09-21T10:42:29.818 INFO:teuthology.orchestra.run.smithi123.stdout: 5 142011 10.48 MB/sec execute 210 sec latency 56.654 ms 2024-09-21T10:42:30.006 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-09-21T10:42:30.006 INFO:teuthology.orchestra.run.smithi123.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472", 2024-09-21T10:42:30.006 INFO:teuthology.orchestra.run.smithi123.stdout: "in_progress": true, 2024-09-21T10:42:30.006 INFO:teuthology.orchestra.run.smithi123.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T10:42:30.006 INFO:teuthology.orchestra.run.smithi123.stdout: "services_complete": [ 2024-09-21T10:42:30.007 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr", 2024-09-21T10:42:30.007 INFO:teuthology.orchestra.run.smithi123.stdout: "mon" 2024-09-21T10:42:30.007 INFO:teuthology.orchestra.run.smithi123.stdout: ], 2024-09-21T10:42:30.007 INFO:teuthology.orchestra.run.smithi123.stdout: "progress": "4/23 daemons upgraded", 2024-09-21T10:42:30.007 INFO:teuthology.orchestra.run.smithi123.stdout: "message": "Currently upgrading mon daemons", 2024-09-21T10:42:30.007 INFO:teuthology.orchestra.run.smithi123.stdout: "is_paused": false 2024-09-21T10:42:30.007 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-09-21T10:42:30.086 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:30 smithi123 ceph-mon[63856]: from='client.44109 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:42:30.086 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:30 smithi123 ceph-mon[63856]: from='client.? 172.21.15.123:0/3957214117' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:42:30.086 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:30 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:30.086 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:30 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:30.086 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:30 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get-or-create", "entity": "mgr.smithi175.txvwrq", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-21T10:42:30.086 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:30 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "mgr services"} : dispatch 2024-09-21T10:42:30.086 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:30 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:42:30.292 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:30 smithi175 ceph-mon[49304]: from='client.44109 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:42:30.292 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:30 smithi175 ceph-mon[49304]: from='client.? 172.21.15.123:0/3957214117' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:42:30.292 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:30 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:30.292 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:30 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:30.292 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:30 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get-or-create", "entity": "mgr.smithi175.txvwrq", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-21T10:42:30.292 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:30 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "mgr services"} : dispatch 2024-09-21T10:42:30.292 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:30 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:42:30.651 INFO:teuthology.orchestra.run.smithi123.stdout:HEALTH_OK 2024-09-21T10:42:30.818 INFO:teuthology.orchestra.run.smithi123.stdout: 5 142345 10.48 MB/sec execute 211 sec latency 72.137 ms 2024-09-21T10:42:31.127 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:31 smithi123 ceph-mon[63856]: Reconfiguring mgr.smithi175.txvwrq (monmap changed)... 2024-09-21T10:42:31.127 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:31 smithi123 ceph-mon[63856]: Reconfiguring daemon mgr.smithi175.txvwrq on smithi175 2024-09-21T10:42:31.127 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:31 smithi123 ceph-mon[63856]: from='client.34170 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:42:31.128 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:31 smithi123 ceph-mon[63856]: pgmap v27: 97 pgs: 97 active+clean; 345 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 10 MiB/s wr, 958 op/s 2024-09-21T10:42:31.128 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:31 smithi123 ceph-mon[63856]: from='client.? 172.21.15.123:0/413016280' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T10:42:31.316 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:31 smithi175 ceph-mon[49304]: Reconfiguring mgr.smithi175.txvwrq (monmap changed)... 2024-09-21T10:42:31.316 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:31 smithi175 ceph-mon[49304]: Reconfiguring daemon mgr.smithi175.txvwrq on smithi175 2024-09-21T10:42:31.316 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:31 smithi175 ceph-mon[49304]: from='client.34170 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:42:31.316 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:31 smithi175 ceph-mon[49304]: pgmap v27: 97 pgs: 97 active+clean; 345 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 10 MiB/s wr, 958 op/s 2024-09-21T10:42:31.316 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:31 smithi175 ceph-mon[49304]: from='client.? 172.21.15.123:0/413016280' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T10:42:31.818 INFO:teuthology.orchestra.run.smithi123.stdout: 5 142572 10.46 MB/sec execute 212 sec latency 56.514 ms 2024-09-21T10:42:32.434 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:32 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:32.434 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:32 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:32.434 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:32 smithi123 ceph-mon[63856]: Reconfiguring mon.smithi175 (monmap changed)... 2024-09-21T10:42:32.434 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:32 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T10:42:32.434 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:32 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T10:42:32.435 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:32 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:42:32.435 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:32 smithi123 ceph-mon[63856]: Reconfiguring daemon mon.smithi175 on smithi175 2024-09-21T10:42:32.649 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:32 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:32.649 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:32 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:32.649 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:32 smithi175 ceph-mon[49304]: Reconfiguring mon.smithi175 (monmap changed)... 2024-09-21T10:42:32.649 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:32 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T10:42:32.650 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:32 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T10:42:32.650 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:32 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:42:32.650 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:32 smithi175 ceph-mon[49304]: Reconfiguring daemon mon.smithi175 on smithi175 2024-09-21T10:42:32.818 INFO:teuthology.orchestra.run.smithi123.stdout: 5 142867 10.45 MB/sec execute 213 sec latency 41.342 ms 2024-09-21T10:42:33.459 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:33 smithi123 ceph-mon[63856]: pgmap v28: 97 pgs: 97 active+clean; 345 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.7 MiB/s wr, 755 op/s 2024-09-21T10:42:33.459 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:33 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:33.459 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:33 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:33.459 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:33 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-21T10:42:33.460 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:33 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:42:33.654 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:33 smithi175 ceph-mon[49304]: pgmap v28: 97 pgs: 97 active+clean; 345 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.7 MiB/s wr, 755 op/s 2024-09-21T10:42:33.654 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:33 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:33.654 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:33 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:33.654 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:33 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-21T10:42:33.654 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:33 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:42:33.819 INFO:teuthology.orchestra.run.smithi123.stdout: 5 143302 10.42 MB/sec execute 214 sec latency 46.777 ms 2024-09-21T10:42:34.481 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:34 smithi123 ceph-mon[63856]: Reconfiguring osd.4 (monmap changed)... 2024-09-21T10:42:34.481 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:34 smithi123 ceph-mon[63856]: Reconfiguring daemon osd.4 on smithi175 2024-09-21T10:42:34.818 INFO:teuthology.orchestra.run.smithi123.stdout: 5 143593 10.40 MB/sec execute 215 sec latency 56.077 ms 2024-09-21T10:42:34.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:34 smithi175 ceph-mon[49304]: Reconfiguring osd.4 (monmap changed)... 2024-09-21T10:42:34.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:34 smithi175 ceph-mon[49304]: Reconfiguring daemon osd.4 on smithi175 2024-09-21T10:42:35.586 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:35 smithi123 ceph-mon[63856]: pgmap v29: 97 pgs: 97 active+clean; 344 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 14 KiB/s rd, 9.8 MiB/s wr, 954 op/s 2024-09-21T10:42:35.586 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:35 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:35.586 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:35 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:35.587 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:35 smithi123 ceph-mon[63856]: Reconfiguring osd.5 (monmap changed)... 2024-09-21T10:42:35.587 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:35 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-21T10:42:35.587 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:35 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:42:35.587 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:35 smithi123 ceph-mon[63856]: Reconfiguring daemon osd.5 on smithi175 2024-09-21T10:42:35.818 INFO:teuthology.orchestra.run.smithi123.stdout: 5 143898 10.38 MB/sec execute 216 sec latency 59.385 ms 2024-09-21T10:42:35.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:35 smithi175 ceph-mon[49304]: pgmap v29: 97 pgs: 97 active+clean; 344 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 14 KiB/s rd, 9.8 MiB/s wr, 954 op/s 2024-09-21T10:42:35.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:35 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:35.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:35 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:35.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:35 smithi175 ceph-mon[49304]: Reconfiguring osd.5 (monmap changed)... 2024-09-21T10:42:35.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:35 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-21T10:42:35.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:35 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:42:35.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:35 smithi175 ceph-mon[49304]: Reconfiguring daemon osd.5 on smithi175 2024-09-21T10:42:36.818 INFO:teuthology.orchestra.run.smithi123.stdout: 5 144299 10.40 MB/sec execute 217 sec latency 59.587 ms 2024-09-21T10:42:37.294 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:37 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:37.294 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:37 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:37.294 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:37 smithi123 ceph-mon[63856]: Reconfiguring osd.6 (monmap changed)... 2024-09-21T10:42:37.295 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:37 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-21T10:42:37.295 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:37 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:42:37.295 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:37 smithi123 ceph-mon[63856]: Reconfiguring daemon osd.6 on smithi175 2024-09-21T10:42:37.295 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:37 smithi123 ceph-mon[63856]: pgmap v30: 97 pgs: 97 active+clean; 344 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.3 KiB/s rd, 5.9 MiB/s wr, 568 op/s 2024-09-21T10:42:37.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:37 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:37.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:37 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:37.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:37 smithi175 ceph-mon[49304]: Reconfiguring osd.6 (monmap changed)... 2024-09-21T10:42:37.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:37 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-21T10:42:37.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:37 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:42:37.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:37 smithi175 ceph-mon[49304]: Reconfiguring daemon osd.6 on smithi175 2024-09-21T10:42:37.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:37 smithi175 ceph-mon[49304]: pgmap v30: 97 pgs: 97 active+clean; 344 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.3 KiB/s rd, 5.9 MiB/s wr, 568 op/s 2024-09-21T10:42:37.818 INFO:teuthology.orchestra.run.smithi123.stdout: 5 144848 10.42 MB/sec execute 218 sec latency 26.540 ms 2024-09-21T10:42:38.819 INFO:teuthology.orchestra.run.smithi123.stdout: 5 145418 10.48 MB/sec execute 219 sec latency 22.375 ms 2024-09-21T10:42:38.932 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:38 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:38.932 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:38 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:38.932 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:38 smithi123 ceph-mon[63856]: Reconfiguring osd.7 (monmap changed)... 2024-09-21T10:42:38.932 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:38 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-21T10:42:38.932 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:38 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:42:38.932 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:38 smithi123 ceph-mon[63856]: Reconfiguring daemon osd.7 on smithi175 2024-09-21T10:42:38.932 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:38 smithi123 ceph-mon[63856]: pgmap v31: 97 pgs: 97 active+clean; 344 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.3 KiB/s rd, 5.9 MiB/s wr, 568 op/s 2024-09-21T10:42:39.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:38 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:39.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:38 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:39.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:38 smithi175 ceph-mon[49304]: Reconfiguring osd.7 (monmap changed)... 2024-09-21T10:42:39.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:38 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-21T10:42:39.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:38 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:42:39.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:38 smithi175 ceph-mon[49304]: Reconfiguring daemon osd.7 on smithi175 2024-09-21T10:42:39.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:38 smithi175 ceph-mon[49304]: pgmap v31: 97 pgs: 97 active+clean; 344 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.3 KiB/s rd, 5.9 MiB/s wr, 568 op/s 2024-09-21T10:42:39.819 INFO:teuthology.orchestra.run.smithi123.stdout: 5 145969 10.50 MB/sec execute 220 sec latency 26.523 ms 2024-09-21T10:42:40.441 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:40 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:40.441 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:40 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:40.441 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:40 smithi123 ceph-mon[63856]: Reconfiguring mds.foofs.smithi175.jrlevw (monmap changed)... 2024-09-21T10:42:40.441 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:40 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi175.jrlevw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-21T10:42:40.441 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:40 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:42:40.441 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:40 smithi123 ceph-mon[63856]: Reconfiguring daemon mds.foofs.smithi175.jrlevw on smithi175 2024-09-21T10:42:40.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:40 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:40.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:40 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:40.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:40 smithi175 ceph-mon[49304]: Reconfiguring mds.foofs.smithi175.jrlevw (monmap changed)... 2024-09-21T10:42:40.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:40 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi175.jrlevw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-21T10:42:40.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:40 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:42:40.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:40 smithi175 ceph-mon[49304]: Reconfiguring daemon mds.foofs.smithi175.jrlevw on smithi175 2024-09-21T10:42:40.819 INFO:teuthology.orchestra.run.smithi123.stdout: 5 146383 10.49 MB/sec execute 221 sec latency 40.031 ms 2024-09-21T10:42:41.819 INFO:teuthology.orchestra.run.smithi123.stdout: 5 146831 10.47 MB/sec execute 222 sec latency 46.759 ms 2024-09-21T10:42:41.956 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:41 smithi123 ceph-mon[63856]: pgmap v32: 97 pgs: 97 active+clean; 346 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.3 MiB/s wr, 807 op/s 2024-09-21T10:42:41.956 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:41 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:41.956 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:41 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:41.957 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:41 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:42:41.957 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:41 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:42:41.957 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:41 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:42:41.957 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:41 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:41.957 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:41 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.smithi123"} : dispatch 2024-09-21T10:42:41.957 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:41 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi123"}]': finished 2024-09-21T10:42:41.957 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:41 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.smithi175"} : dispatch 2024-09-21T10:42:41.957 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:41 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi175"}]': finished 2024-09-21T10:42:42.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:41 smithi175 ceph-mon[49304]: pgmap v32: 97 pgs: 97 active+clean; 346 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.3 MiB/s wr, 807 op/s 2024-09-21T10:42:42.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:41 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:42.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:41 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:42.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:41 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:42:42.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:41 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:42:42.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:41 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:42:42.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:41 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:42.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:41 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.smithi123"} : dispatch 2024-09-21T10:42:42.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:41 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi123"}]': finished 2024-09-21T10:42:42.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:41 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.smithi175"} : dispatch 2024-09-21T10:42:42.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:41 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi175"}]': finished 2024-09-21T10:42:42.819 INFO:teuthology.orchestra.run.smithi123.stdout: 5 147190 10.46 MB/sec execute 223 sec latency 41.921 ms 2024-09-21T10:42:42.990 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:42 smithi123 ceph-mon[63856]: Upgrade: Setting container_image for all mon 2024-09-21T10:42:42.990 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:42 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:42:43.024 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:42 smithi175 ceph-mon[49304]: Upgrade: Setting container_image for all mon 2024-09-21T10:42:43.024 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:42 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:42:43.819 INFO:teuthology.orchestra.run.smithi123.stdout: 5 147639 10.48 MB/sec execute 224 sec latency 35.596 ms 2024-09-21T10:42:44.003 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:43 smithi123 ceph-mon[63856]: pgmap v33: 97 pgs: 97 active+clean; 346 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.5 MiB/s wr, 438 op/s 2024-09-21T10:42:44.003 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:43 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:44.003 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:43 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi123", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-21T10:42:44.003 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:43 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:42:44.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:43 smithi175 ceph-mon[49304]: pgmap v33: 97 pgs: 97 active+clean; 346 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.5 MiB/s wr, 438 op/s 2024-09-21T10:42:44.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:43 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:44.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:43 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi123", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-21T10:42:44.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:43 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:42:44.757 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:44 smithi123 ceph-mon[63856]: Upgrade: Updating crash.smithi123 (1/2) 2024-09-21T10:42:44.757 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:44 smithi123 ceph-mon[63856]: Deploying daemon crash.smithi123 on smithi123 2024-09-21T10:42:44.819 INFO:teuthology.orchestra.run.smithi123.stdout: 5 148114 10.47 MB/sec execute 225 sec latency 47.083 ms 2024-09-21T10:42:45.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:44 smithi175 ceph-mon[49304]: Upgrade: Updating crash.smithi123 (1/2) 2024-09-21T10:42:45.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:44 smithi175 ceph-mon[49304]: Deploying daemon crash.smithi123 on smithi123 2024-09-21T10:42:45.779 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:45 smithi123 ceph-mon[63856]: pgmap v34: 97 pgs: 97 active+clean; 336 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 7.3 MiB/s wr, 729 op/s 2024-09-21T10:42:45.819 INFO:teuthology.orchestra.run.smithi123.stdout: 5 148532 10.50 MB/sec execute 226 sec latency 39.370 ms 2024-09-21T10:42:46.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:45 smithi175 ceph-mon[49304]: pgmap v34: 97 pgs: 97 active+clean; 336 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 7.3 MiB/s wr, 729 op/s 2024-09-21T10:42:46.819 INFO:teuthology.orchestra.run.smithi123.stdout: 5 148924 10.53 MB/sec execute 227 sec latency 51.733 ms 2024-09-21T10:42:47.801 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:47 smithi123 ceph-mon[63856]: pgmap v35: 97 pgs: 97 active+clean; 336 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.2 MiB/s wr, 530 op/s 2024-09-21T10:42:47.819 INFO:teuthology.orchestra.run.smithi123.stdout: 5 149410 10.54 MB/sec execute 228 sec latency 40.759 ms 2024-09-21T10:42:48.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:47 smithi175 ceph-mon[49304]: pgmap v35: 97 pgs: 97 active+clean; 336 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.2 MiB/s wr, 530 op/s 2024-09-21T10:42:48.819 INFO:teuthology.orchestra.run.smithi123.stdout: 5 149676 10.51 MB/sec execute 229 sec latency 66.642 ms 2024-09-21T10:42:49.819 INFO:teuthology.orchestra.run.smithi123.stdout: 5 149913 10.49 MB/sec execute 230 sec latency 68.032 ms 2024-09-21T10:42:49.867 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:49 smithi123 ceph-mon[63856]: pgmap v36: 97 pgs: 97 active+clean; 336 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.2 MiB/s wr, 530 op/s 2024-09-21T10:42:49.867 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:49 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:49.867 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:49 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:50.029 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:49 smithi175 ceph-mon[49304]: pgmap v36: 97 pgs: 97 active+clean; 336 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.2 MiB/s wr, 530 op/s 2024-09-21T10:42:50.029 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:49 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:50.029 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:49 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:50.819 INFO:teuthology.orchestra.run.smithi123.stdout: 5 150436 10.48 MB/sec execute 231 sec latency 47.311 ms 2024-09-21T10:42:51.819 INFO:teuthology.orchestra.run.smithi123.stdout: 5 150974 10.49 MB/sec execute 232 sec latency 97.501 ms 2024-09-21T10:42:51.888 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:51 smithi123 ceph-mon[63856]: pgmap v37: 97 pgs: 97 active+clean; 347 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 8.0 MiB/s wr, 770 op/s 2024-09-21T10:42:51.888 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:51 smithi123 ceph-mon[63856]: Upgrade: Updating crash.smithi175 (2/2) 2024-09-21T10:42:51.888 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:51 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:51.888 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:51 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi175", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-21T10:42:51.888 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:51 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:42:51.889 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:51 smithi123 ceph-mon[63856]: Deploying daemon crash.smithi175 on smithi175 2024-09-21T10:42:52.053 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:51 smithi175 ceph-mon[49304]: pgmap v37: 97 pgs: 97 active+clean; 347 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 8.0 MiB/s wr, 770 op/s 2024-09-21T10:42:52.053 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:51 smithi175 ceph-mon[49304]: Upgrade: Updating crash.smithi175 (2/2) 2024-09-21T10:42:52.054 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:51 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:52.054 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:51 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi175", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-21T10:42:52.054 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:51 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:42:52.054 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:51 smithi175 ceph-mon[49304]: Deploying daemon crash.smithi175 on smithi175 2024-09-21T10:42:52.819 INFO:teuthology.orchestra.run.smithi123.stdout: 5 151487 10.51 MB/sec execute 233 sec latency 41.039 ms 2024-09-21T10:42:53.819 INFO:teuthology.orchestra.run.smithi123.stdout: 5 152019 10.55 MB/sec execute 234 sec latency 29.605 ms 2024-09-21T10:42:53.910 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:53 smithi123 ceph-mon[63856]: pgmap v38: 97 pgs: 97 active+clean; 347 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.5 MiB/s wr, 531 op/s 2024-09-21T10:42:53.961 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:53 smithi175 ceph-mon[49304]: pgmap v38: 97 pgs: 97 active+clean; 347 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.5 MiB/s wr, 531 op/s 2024-09-21T10:42:54.819 INFO:teuthology.orchestra.run.smithi123.stdout: 5 152467 10.58 MB/sec execute 235 sec latency 26.259 ms 2024-09-21T10:42:55.682 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:55 smithi123 ceph-mon[63856]: pgmap v39: 97 pgs: 97 active+clean; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.0 MiB/s wr, 806 op/s 2024-09-21T10:42:55.682 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:55 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:55.682 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:55 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:55.682 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:55 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:42:55.819 INFO:teuthology.orchestra.run.smithi123.stdout: 5 153087 10.60 MB/sec execute 236 sec latency 53.061 ms 2024-09-21T10:42:55.935 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:55 smithi175 ceph-mon[49304]: pgmap v39: 97 pgs: 97 active+clean; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.0 MiB/s wr, 806 op/s 2024-09-21T10:42:55.935 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:55 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:55.935 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:55 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:55.935 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:55 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:42:56.820 INFO:teuthology.orchestra.run.smithi123.stdout: 5 153605 10.60 MB/sec execute 237 sec latency 35.457 ms 2024-09-21T10:42:57.696 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:57 smithi123 ceph-mon[63856]: pgmap v40: 97 pgs: 97 active+clean; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.2 MiB/s wr, 514 op/s 2024-09-21T10:42:57.696 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:57 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:42:57.704 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:57 smithi175 ceph-mon[49304]: pgmap v40: 97 pgs: 97 active+clean; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.2 MiB/s wr, 514 op/s 2024-09-21T10:42:57.704 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:57 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:42:57.819 INFO:teuthology.orchestra.run.smithi123.stdout: 5 153937 10.57 MB/sec execute 238 sec latency 65.686 ms 2024-09-21T10:42:58.820 INFO:teuthology.orchestra.run.smithi123.stdout: 5 154244 10.56 MB/sec execute 239 sec latency 49.632 ms 2024-09-21T10:42:59.739 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:59 smithi123 ceph-mon[63856]: pgmap v41: 97 pgs: 97 active+clean; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.2 MiB/s wr, 514 op/s 2024-09-21T10:42:59.739 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:59 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:59.739 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:42:59 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:42:59.819 INFO:teuthology.orchestra.run.smithi123.stdout: 5 154602 10.56 MB/sec execute 240 sec latency 60.513 ms 2024-09-21T10:43:00.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:59 smithi175 ceph-mon[49304]: pgmap v41: 97 pgs: 97 active+clean; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.2 MiB/s wr, 514 op/s 2024-09-21T10:43:00.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:59 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:43:00.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:42:59 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:43:00.820 INFO:teuthology.orchestra.run.smithi123.stdout: 5 155288 10.59 MB/sec execute 241 sec latency 43.138 ms 2024-09-21T10:43:01.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:01 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:43:01.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:01 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:43:01.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:01 smithi175 ceph-mon[49304]: pgmap v42: 97 pgs: 97 active+clean; 345 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 8.2 MiB/s wr, 801 op/s 2024-09-21T10:43:01.263 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:01 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:43:01.263 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:01 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:43:01.263 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:01 smithi123 ceph-mon[63856]: pgmap v42: 97 pgs: 97 active+clean; 345 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 8.2 MiB/s wr, 801 op/s 2024-09-21T10:43:01.265 INFO:teuthology.orchestra.run.smithi123.stdout:true 2024-09-21T10:43:01.820 INFO:teuthology.orchestra.run.smithi123.stdout: 5 155740 10.61 MB/sec execute 242 sec latency 51.382 ms 2024-09-21T10:43:02.276 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:02 smithi123 ceph-mon[63856]: from='client.44125 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:43:02.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:02 smithi175 ceph-mon[49304]: from='client.44125 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:43:02.413 INFO:teuthology.orchestra.run.smithi123.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T10:43:02.413 INFO:teuthology.orchestra.run.smithi123.stdout:alertmanager.smithi123 smithi123 *:9093,9094 running (3m) 71s ago 12m 19.7M - 0.27.0 11f11916f8cd 5d550bd97de5 2024-09-21T10:43:02.413 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi123 smithi123 starting - - - - 2024-09-21T10:43:02.413 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi175 smithi175 running (7s) 2s ago 12m 6895k - 19.3.0-5103-g5c1cdc3c 8f4d013db793 fb1ca999ccfd 2024-09-21T10:43:02.413 INFO:teuthology.orchestra.run.smithi123.stdout:grafana.smithi123 smithi123 *:3000 running (3m) 71s ago 12m 43.4M - dad864ee21e9 771df81763ba 2024-09-21T10:43:02.413 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi123.uqoyow smithi123 running (6m) 71s ago 6m 40.4M - 17.2.0 e1d6a67b021e f0a21eaac266 2024-09-21T10:43:02.413 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi175.jrlevw smithi175 running (6m) 2s ago 6m 13.5M - 17.2.0 e1d6a67b021e b21b16f4bc50 2024-09-21T10:43:02.413 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi123.stnuit smithi123 *:8443,9283,8765 running (2m) 71s ago 13m 489M - 19.3.0-5103-g5c1cdc3c 8f4d013db793 6812cc5cb56b 2024-09-21T10:43:02.413 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi175.txvwrq smithi175 *:8443,9283,8765 running (104s) 2s ago 11m 453M - 19.3.0-5103-g5c1cdc3c 8f4d013db793 304f0408075e 2024-09-21T10:43:02.413 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi123 smithi123 running (88s) 71s ago 14m 30.6M 2048M 19.3.0-5103-g5c1cdc3c 8f4d013db793 6b95207a2d80 2024-09-21T10:43:02.413 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi175 smithi175 running (64s) 2s ago 11m 36.7M 2048M 19.3.0-5103-g5c1cdc3c 8f4d013db793 3abee2d37f0f 2024-09-21T10:43:02.413 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.0.0.smithi123.ddatpr smithi123 *:2049 running (6m) 71s ago 6m 275M - 4.0 e1d6a67b021e 36f8cb2281eb 2024-09-21T10:43:02.413 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.1.0.smithi175.wpawer smithi175 *:2049 running (6m) 2s ago 6m 90.3M - 4.0 e1d6a67b021e 2200aeed9200 2024-09-21T10:43:02.414 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi123 smithi123 *:9100 running (3m) 71s ago 12m 10.2M - 1.7.0 72c9c2088986 ca7d0b29af81 2024-09-21T10:43:02.414 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi175 smithi175 *:9100 running (2m) 2s ago 11m 11.1M - 1.7.0 72c9c2088986 46c4f6b7c149 2024-09-21T10:43:02.414 INFO:teuthology.orchestra.run.smithi123.stdout:osd.0 smithi123 running (11m) 71s ago 11m 449M 1936M 17.2.0 e1d6a67b021e a94edf88c456 2024-09-21T10:43:02.414 INFO:teuthology.orchestra.run.smithi123.stdout:osd.1 smithi123 running (10m) 71s ago 10m 404M 1936M 17.2.0 e1d6a67b021e 59aee515b0f8 2024-09-21T10:43:02.414 INFO:teuthology.orchestra.run.smithi123.stdout:osd.2 smithi123 running (10m) 71s ago 10m 293M 1936M 17.2.0 e1d6a67b021e 7db93f8ce5b5 2024-09-21T10:43:02.414 INFO:teuthology.orchestra.run.smithi123.stdout:osd.3 smithi123 running (9m) 71s ago 9m 420M 1936M 17.2.0 e1d6a67b021e c1672dea7b1f 2024-09-21T10:43:02.414 INFO:teuthology.orchestra.run.smithi123.stdout:osd.4 smithi175 running (9m) 2s ago 9m 498M 2704M 17.2.0 e1d6a67b021e 48fe728c259e 2024-09-21T10:43:02.414 INFO:teuthology.orchestra.run.smithi123.stdout:osd.5 smithi175 running (8m) 2s ago 8m 331M 2704M 17.2.0 e1d6a67b021e e270d5a21b14 2024-09-21T10:43:02.414 INFO:teuthology.orchestra.run.smithi123.stdout:osd.6 smithi175 running (8m) 2s ago 8m 517M 2704M 17.2.0 e1d6a67b021e 9bcbf018eb20 2024-09-21T10:43:02.414 INFO:teuthology.orchestra.run.smithi123.stdout:osd.7 smithi175 running (8m) 2s ago 8m 383M 2704M 17.2.0 e1d6a67b021e e9b8f1f90994 2024-09-21T10:43:02.414 INFO:teuthology.orchestra.run.smithi123.stdout:prometheus.smithi123 smithi123 *:9095 running (110s) 71s ago 11m 46.0M - 2.51.0 1d3b7f56885b 37f944c9de88 2024-09-21T10:43:02.820 INFO:teuthology.orchestra.run.smithi123.stdout: 5 156332 10.65 MB/sec execute 243 sec latency 49.179 ms 2024-09-21T10:43:03.138 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:03 smithi123 ceph-mon[63856]: from='client.44129 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:43:03.138 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:03 smithi123 ceph-mon[63856]: pgmap v43: 97 pgs: 97 active+clean; 345 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 5.5 MiB/s wr, 561 op/s 2024-09-21T10:43:03.138 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:03 smithi123 ceph-mon[63856]: from='client.44133 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:43:03.145 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-09-21T10:43:03.146 INFO:teuthology.orchestra.run.smithi123.stdout: "mon": { 2024-09-21T10:43:03.146 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev)": 2 2024-09-21T10:43:03.146 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:43:03.146 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr": { 2024-09-21T10:43:03.146 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev)": 2 2024-09-21T10:43:03.146 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:43:03.146 INFO:teuthology.orchestra.run.smithi123.stdout: "osd": { 2024-09-21T10:43:03.146 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-21T10:43:03.146 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:43:03.146 INFO:teuthology.orchestra.run.smithi123.stdout: "mds": { 2024-09-21T10:43:03.146 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T10:43:03.146 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:43:03.146 INFO:teuthology.orchestra.run.smithi123.stdout: "overall": { 2024-09-21T10:43:03.147 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-09-21T10:43:03.147 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev)": 4 2024-09-21T10:43:03.147 INFO:teuthology.orchestra.run.smithi123.stdout: } 2024-09-21T10:43:03.147 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-09-21T10:43:03.332 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:03 smithi175 ceph-mon[49304]: from='client.44129 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:43:03.332 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:03 smithi175 ceph-mon[49304]: pgmap v43: 97 pgs: 97 active+clean; 345 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 5.5 MiB/s wr, 561 op/s 2024-09-21T10:43:03.332 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:03 smithi175 ceph-mon[49304]: from='client.44133 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:43:03.741 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-09-21T10:43:03.741 INFO:teuthology.orchestra.run.smithi123.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472", 2024-09-21T10:43:03.741 INFO:teuthology.orchestra.run.smithi123.stdout: "in_progress": true, 2024-09-21T10:43:03.741 INFO:teuthology.orchestra.run.smithi123.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T10:43:03.741 INFO:teuthology.orchestra.run.smithi123.stdout: "services_complete": [ 2024-09-21T10:43:03.741 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr", 2024-09-21T10:43:03.741 INFO:teuthology.orchestra.run.smithi123.stdout: "mon" 2024-09-21T10:43:03.741 INFO:teuthology.orchestra.run.smithi123.stdout: ], 2024-09-21T10:43:03.741 INFO:teuthology.orchestra.run.smithi123.stdout: "progress": "5/23 daemons upgraded", 2024-09-21T10:43:03.741 INFO:teuthology.orchestra.run.smithi123.stdout: "message": "Currently upgrading crash daemons", 2024-09-21T10:43:03.741 INFO:teuthology.orchestra.run.smithi123.stdout: "is_paused": false 2024-09-21T10:43:03.741 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-09-21T10:43:03.820 INFO:teuthology.orchestra.run.smithi123.stdout: 5 156772 10.65 MB/sec execute 244 sec latency 62.900 ms 2024-09-21T10:43:04.148 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:04 smithi123 ceph-mon[63856]: from='client.? 172.21.15.123:0/2989965775' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:43:04.148 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:04 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:43:04.148 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:04 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:43:04.148 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:04 smithi123 ceph-mon[63856]: from='client.44141 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:43:04.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:04 smithi175 ceph-mon[49304]: from='client.? 172.21.15.123:0/2989965775' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:43:04.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:04 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:43:04.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:04 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:43:04.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:04 smithi175 ceph-mon[49304]: from='client.44141 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:43:04.456 INFO:teuthology.orchestra.run.smithi123.stdout:HEALTH_OK 2024-09-21T10:43:04.820 INFO:teuthology.orchestra.run.smithi123.stdout: 5 157227 10.65 MB/sec execute 245 sec latency 38.338 ms 2024-09-21T10:43:05.156 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:05 smithi123 ceph-mon[63856]: pgmap v44: 97 pgs: 97 active+clean; 352 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.6 MiB/s wr, 846 op/s 2024-09-21T10:43:05.156 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:05 smithi123 ceph-mon[63856]: from='client.? 172.21.15.123:0/3117535367' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T10:43:05.156 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:05 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:43:05.156 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:05 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:43:05.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:05 smithi175 ceph-mon[49304]: pgmap v44: 97 pgs: 97 active+clean; 352 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.6 MiB/s wr, 846 op/s 2024-09-21T10:43:05.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:05 smithi175 ceph-mon[49304]: from='client.? 172.21.15.123:0/3117535367' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T10:43:05.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:05 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:43:05.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:05 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:43:05.820 INFO:teuthology.orchestra.run.smithi123.stdout: 5 157664 10.63 MB/sec execute 246 sec latency 42.516 ms 2024-09-21T10:43:06.422 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:06 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:43:06.423 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:06 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:43:06.820 INFO:teuthology.orchestra.run.smithi123.stdout: 5 158163 10.65 MB/sec execute 247 sec latency 68.865 ms 2024-09-21T10:43:06.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:06 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:43:06.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:06 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:43:07.462 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:07 smithi123 ceph-mon[63856]: pgmap v45: 97 pgs: 97 active+clean; 352 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 6.2 MiB/s wr, 572 op/s 2024-09-21T10:43:07.820 INFO:teuthology.orchestra.run.smithi123.stdout: 5 158554 10.65 MB/sec execute 248 sec latency 56.777 ms 2024-09-21T10:43:07.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:07 smithi175 ceph-mon[49304]: pgmap v45: 97 pgs: 97 active+clean; 352 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 6.2 MiB/s wr, 572 op/s 2024-09-21T10:43:08.820 INFO:teuthology.orchestra.run.smithi123.stdout: 5 158956 10.65 MB/sec execute 249 sec latency 39.372 ms 2024-09-21T10:43:09.494 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:09 smithi123 ceph-mon[63856]: pgmap v46: 97 pgs: 97 active+clean; 352 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 6.2 MiB/s wr, 572 op/s 2024-09-21T10:43:09.820 INFO:teuthology.orchestra.run.smithi123.stdout: 5 159262 10.65 MB/sec execute 250 sec latency 34.570 ms 2024-09-21T10:43:09.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:09 smithi175 ceph-mon[49304]: pgmap v46: 97 pgs: 97 active+clean; 352 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 6.2 MiB/s wr, 572 op/s 2024-09-21T10:43:10.820 INFO:teuthology.orchestra.run.smithi123.stdout: 5 159594 10.67 MB/sec execute 251 sec latency 58.409 ms 2024-09-21T10:43:11.567 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:11 smithi123 ceph-mon[63856]: pgmap v47: 97 pgs: 97 active+clean; 344 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.6 MiB/s wr, 846 op/s 2024-09-21T10:43:11.820 INFO:teuthology.orchestra.run.smithi123.stdout: 5 159919 10.66 MB/sec execute 252 sec latency 53.917 ms 2024-09-21T10:43:11.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:11 smithi175 ceph-mon[49304]: pgmap v47: 97 pgs: 97 active+clean; 344 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.6 MiB/s wr, 846 op/s 2024-09-21T10:43:12.820 INFO:teuthology.orchestra.run.smithi123.stdout: 5 160170 10.64 MB/sec execute 253 sec latency 45.105 ms 2024-09-21T10:43:13.353 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:13 smithi123 ceph-mon[63856]: pgmap v48: 97 pgs: 97 active+clean; 344 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.6 MiB/s wr, 559 op/s 2024-09-21T10:43:13.354 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:13 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:43:13.354 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:13 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:43:13.354 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:13 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:43:13.354 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:13 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:43:13.354 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:13 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:43:13.354 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:13 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:43:13.354 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:13 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:43:13.354 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:13 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:43:13.354 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:13 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:43:13.354 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:13 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:43:13.354 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:13 smithi123 ceph-mon[63856]: Upgrade: Setting container_image for all crash 2024-09-21T10:43:13.355 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:13 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:43:13.355 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:13 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi123"} : dispatch 2024-09-21T10:43:13.355 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:13 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi123"}]': finished 2024-09-21T10:43:13.355 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:13 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi175"} : dispatch 2024-09-21T10:43:13.355 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:13 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi175"}]': finished 2024-09-21T10:43:13.355 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:13 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-09-21T10:43:13.355 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:13 smithi123 ceph-mon[63856]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-21T10:43:13.355 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:13 smithi123 ceph-mon[63856]: Upgrade: osd.0 is safe to restart 2024-09-21T10:43:13.355 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:13 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:43:13.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:13 smithi175 ceph-mon[49304]: pgmap v48: 97 pgs: 97 active+clean; 344 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.6 MiB/s wr, 559 op/s 2024-09-21T10:43:13.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:13 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:43:13.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:13 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:43:13.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:13 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:43:13.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:13 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:43:13.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:13 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:43:13.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:13 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:43:13.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:13 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:43:13.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:13 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:43:13.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:13 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:43:13.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:13 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:43:13.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:13 smithi175 ceph-mon[49304]: Upgrade: Setting container_image for all crash 2024-09-21T10:43:13.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:13 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:43:13.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:13 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi123"} : dispatch 2024-09-21T10:43:13.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:13 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi123"}]': finished 2024-09-21T10:43:13.625 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:13 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi175"} : dispatch 2024-09-21T10:43:13.625 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:13 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi175"}]': finished 2024-09-21T10:43:13.625 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:13 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-09-21T10:43:13.625 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:13 smithi175 ceph-mon[49304]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-21T10:43:13.625 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:13 smithi175 ceph-mon[49304]: Upgrade: osd.0 is safe to restart 2024-09-21T10:43:13.625 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:13 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:43:13.820 INFO:teuthology.orchestra.run.smithi123.stdout: 5 160687 10.64 MB/sec execute 254 sec latency 39.451 ms 2024-09-21T10:43:14.820 INFO:teuthology.orchestra.run.smithi123.stdout: 5 161323 10.64 MB/sec execute 255 sec latency 28.014 ms 2024-09-21T10:43:15.306 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:15 smithi123 ceph-mon[63856]: pgmap v49: 97 pgs: 97 active+clean; 348 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 8.0 MiB/s wr, 765 op/s 2024-09-21T10:43:15.306 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:15 smithi123 ceph-mon[63856]: mgrmap e41: smithi123.stnuit(active, since 92s), standbys: smithi175.txvwrq 2024-09-21T10:43:15.306 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:15 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:43:15.306 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:15 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-21T10:43:15.306 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:15 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:43:15.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:15 smithi175 ceph-mon[49304]: pgmap v49: 97 pgs: 97 active+clean; 348 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 8.0 MiB/s wr, 765 op/s 2024-09-21T10:43:15.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:15 smithi175 ceph-mon[49304]: mgrmap e41: smithi123.stnuit(active, since 92s), standbys: smithi175.txvwrq 2024-09-21T10:43:15.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:15 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:43:15.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:15 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-21T10:43:15.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:15 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:43:15.820 INFO:teuthology.orchestra.run.smithi123.stdout: 5 162100 10.69 MB/sec execute 256 sec latency 24.308 ms 2024-09-21T10:43:16.372 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:16 smithi123 ceph-mon[63856]: Upgrade: Updating osd.0 2024-09-21T10:43:16.372 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:16 smithi123 ceph-mon[63856]: Deploying daemon osd.0 on smithi123 2024-09-21T10:43:16.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:16 smithi175 ceph-mon[49304]: Upgrade: Updating osd.0 2024-09-21T10:43:16.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:16 smithi175 ceph-mon[49304]: Deploying daemon osd.0 on smithi123 2024-09-21T10:43:16.820 INFO:teuthology.orchestra.run.smithi123.stdout: 5 162843 10.74 MB/sec execute 257 sec latency 31.290 ms 2024-09-21T10:43:17.385 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:17 smithi123 ceph-mon[63856]: pgmap v50: 97 pgs: 97 active+clean; 348 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.9 MiB/s wr, 480 op/s 2024-09-21T10:43:17.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:17 smithi175 ceph-mon[49304]: pgmap v50: 97 pgs: 97 active+clean; 348 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.9 MiB/s wr, 480 op/s 2024-09-21T10:43:17.820 INFO:teuthology.orchestra.run.smithi123.stdout: 5 163366 10.77 MB/sec execute 258 sec latency 62.672 ms 2024-09-21T10:43:18.820 INFO:teuthology.orchestra.run.smithi123.stdout: 5 163680 10.76 MB/sec execute 259 sec latency 56.008 ms 2024-09-21T10:43:19.435 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:19 smithi123 ceph-mon[63856]: pgmap v51: 97 pgs: 97 active+clean; 348 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.9 MiB/s wr, 480 op/s 2024-09-21T10:43:19.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:19 smithi175 ceph-mon[49304]: pgmap v51: 97 pgs: 97 active+clean; 348 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.9 MiB/s wr, 480 op/s 2024-09-21T10:43:19.686 INFO:journalctl@ceph.osd.0.smithi123.stdout:Sep 21 10:43:19 smithi123 systemd[1]: Stopping Ceph osd.0 for 3ccd15e6-7804-11ef-baf6-efdc52797490... 2024-09-21T10:43:19.820 INFO:teuthology.orchestra.run.smithi123.stdout: 5 163877 10.73 MB/sec execute 260 sec latency 53.311 ms 2024-09-21T10:43:19.939 INFO:journalctl@ceph.osd.0.smithi123.stdout:Sep 21 10:43:19 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-0[34212]: 2024-09-21T10:43:19.908+0000 7f2fc287f700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:43:19.939 INFO:journalctl@ceph.osd.0.smithi123.stdout:Sep 21 10:43:19 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-0[34212]: 2024-09-21T10:43:19.908+0000 7f2fc287f700 -1 osd.0 76 *** Got signal Terminated *** 2024-09-21T10:43:19.939 INFO:journalctl@ceph.osd.0.smithi123.stdout:Sep 21 10:43:19 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-0[34212]: 2024-09-21T10:43:19.908+0000 7f2fc287f700 -1 osd.0 76 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T10:43:20.451 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:20 smithi123 ceph-mon[63856]: osd.0 marked itself down and dead 2024-09-21T10:43:20.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:20 smithi175 ceph-mon[49304]: osd.0 marked itself down and dead 2024-09-21T10:43:20.820 INFO:teuthology.orchestra.run.smithi123.stdout: 5 164082 10.71 MB/sec execute 261 sec latency 411.417 ms 2024-09-21T10:43:21.262 INFO:journalctl@ceph.osd.0.smithi123.stdout:Sep 21 10:43:20 smithi123 podman[68163]: 2024-09-21 10:43:20.985348622 +0000 UTC m=+1.205056972 container died a94edf88c456a09afeb86432f36d8a2d881438c3c7047270f4d6a721f30598f7 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , architecture=x86_64, ceph=True, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, com.redhat.component=centos-stream-container, RELEASE=HEAD, distribution-scope=public, io.openshift.tags=base centos centos-stream, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, io.buildah.version=1.19.8, vcs-type=git) 2024-09-21T10:43:21.512 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:21 smithi123 ceph-mon[63856]: pgmap v52: 97 pgs: 97 active+clean; 352 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.7 MiB/s wr, 845 op/s 2024-09-21T10:43:21.512 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:21 smithi123 ceph-mon[63856]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T10:43:21.512 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:21 smithi123 ceph-mon[63856]: osdmap e77: 8 total, 7 up, 8 in 2024-09-21T10:43:21.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:21 smithi175 ceph-mon[49304]: pgmap v52: 97 pgs: 97 active+clean; 352 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.7 MiB/s wr, 845 op/s 2024-09-21T10:43:21.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:21 smithi175 ceph-mon[49304]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T10:43:21.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:21 smithi175 ceph-mon[49304]: osdmap e77: 8 total, 7 up, 8 in 2024-09-21T10:43:21.804 INFO:journalctl@ceph.osd.0.smithi123.stdout:Sep 21 10:43:21 smithi123 podman[68163]: 2024-09-21 10:43:21.584798768 +0000 UTC m=+1.804507118 container cleanup a94edf88c456a09afeb86432f36d8a2d881438c3c7047270f4d6a721f30598f7 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-0, GIT_CLEAN=True, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, io.k8s.display-name=CentOS Stream 8, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, architecture=x86_64, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, name=centos-stream, build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD) 2024-09-21T10:43:21.804 INFO:journalctl@ceph.osd.0.smithi123.stdout:Sep 21 10:43:21 smithi123 bash[68163]: ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-0 2024-09-21T10:43:21.820 INFO:teuthology.orchestra.run.smithi123.stdout: 5 164283 10.68 MB/sec execute 262 sec latency 877.180 ms 2024-09-21T10:43:22.058 INFO:journalctl@ceph.osd.0.smithi123.stdout:Sep 21 10:43:21 smithi123 podman[68176]: 2024-09-21 10:43:21.899453964 +0000 UTC m=+0.917256826 container remove a94edf88c456a09afeb86432f36d8a2d881438c3c7047270f4d6a721f30598f7 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-0, GIT_BRANCH=HEAD, version=8, maintainer=Guillaume Abrioux , vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, io.openshift.expose-services=, architecture=x86_64, vendor=Red Hat, Inc., release=754, ceph=True, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-09-21T10:43:22.314 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:22 smithi123 ceph-mon[63856]: osdmap e78: 8 total, 7 up, 8 in 2024-09-21T10:43:22.568 INFO:journalctl@ceph.osd.0.smithi123.stdout:Sep 21 10:43:22 smithi123 podman[68266]: 2024-09-21 10:43:22.390708322 +0000 UTC m=+0.019949598 image pull 8f4d013db793795fef09ff5edf96e6b42ba8cd2c9158b2f0b7a5d7c1e657c063 quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472 2024-09-21T10:43:22.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:22 smithi175 ceph-mon[49304]: osdmap e78: 8 total, 7 up, 8 in 2024-09-21T10:43:22.820 INFO:teuthology.orchestra.run.smithi123.stdout: 5 164591 10.66 MB/sec execute 263 sec latency 53.419 ms 2024-09-21T10:43:22.825 INFO:journalctl@ceph.osd.0.smithi123.stdout:Sep 21 10:43:22 smithi123 podman[68266]: 2024-09-21 10:43:22.643853791 +0000 UTC m=+0.273095064 container create 740bbe17c92976c9e899f278774c24bef1f74fe9c4485617db81b450d311c1a3 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-0-deactivate, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T10:43:23.076 INFO:journalctl@ceph.osd.0.smithi123.stdout:Sep 21 10:43:23 smithi123 podman[68266]: 2024-09-21 10:43:23.034233451 +0000 UTC m=+0.663474730 container init 740bbe17c92976c9e899f278774c24bef1f74fe9c4485617db81b450d311c1a3 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-0-deactivate, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, org.label-schema.license=GPLv2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T10:43:23.076 INFO:journalctl@ceph.osd.0.smithi123.stdout:Sep 21 10:43:23 smithi123 podman[68266]: 2024-09-21 10:43:23.048136592 +0000 UTC m=+0.677377865 container start 740bbe17c92976c9e899f278774c24bef1f74fe9c4485617db81b450d311c1a3 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-0-deactivate, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3) 2024-09-21T10:43:23.327 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:23 smithi123 ceph-mon[63856]: pgmap v55: 97 pgs: 13 stale+active+clean, 84 active+clean; 352 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 5.7 MiB/s wr, 548 op/s 2024-09-21T10:43:23.327 INFO:journalctl@ceph.osd.0.smithi123.stdout:Sep 21 10:43:23 smithi123 podman[68266]: 2024-09-21 10:43:23.176315451 +0000 UTC m=+0.805556724 container attach 740bbe17c92976c9e899f278774c24bef1f74fe9c4485617db81b450d311c1a3 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-0-deactivate, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=) 2024-09-21T10:43:23.583 INFO:journalctl@ceph.osd.0.smithi123.stdout:Sep 21 10:43:23 smithi123 podman[68266]: 2024-09-21 10:43:23.407172909 +0000 UTC m=+1.036414177 container died 740bbe17c92976c9e899f278774c24bef1f74fe9c4485617db81b450d311c1a3 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-0-deactivate, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-09-21T10:43:23.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:23 smithi175 ceph-mon[49304]: pgmap v55: 97 pgs: 13 stale+active+clean, 84 active+clean; 352 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 5.7 MiB/s wr, 548 op/s 2024-09-21T10:43:23.821 INFO:teuthology.orchestra.run.smithi123.stdout: 5 164931 10.65 MB/sec execute 264 sec latency 41.866 ms 2024-09-21T10:43:24.357 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:24 smithi123 ceph-mon[63856]: Health check failed: Degraded data redundancy: 192/1464 objects degraded (13.115%), 29 pgs degraded (PG_DEGRADED) 2024-09-21T10:43:24.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:24 smithi175 ceph-mon[49304]: Health check failed: Degraded data redundancy: 192/1464 objects degraded (13.115%), 29 pgs degraded (PG_DEGRADED) 2024-09-21T10:43:24.636 INFO:journalctl@ceph.osd.0.smithi123.stdout:Sep 21 10:43:24 smithi123 podman[68288]: 2024-09-21 10:43:24.37484956 +0000 UTC m=+0.957341231 container remove 740bbe17c92976c9e899f278774c24bef1f74fe9c4485617db81b450d311c1a3 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-0-deactivate, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, ceph=True, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T10:43:24.636 INFO:journalctl@ceph.osd.0.smithi123.stdout:Sep 21 10:43:24 smithi123 systemd[1]: ceph-3ccd15e6-7804-11ef-baf6-efdc52797490@osd.0.service: Deactivated successfully. 2024-09-21T10:43:24.636 INFO:journalctl@ceph.osd.0.smithi123.stdout:Sep 21 10:43:24 smithi123 systemd[1]: Stopped Ceph osd.0 for 3ccd15e6-7804-11ef-baf6-efdc52797490. 2024-09-21T10:43:24.637 INFO:journalctl@ceph.osd.0.smithi123.stdout:Sep 21 10:43:24 smithi123 systemd[1]: ceph-3ccd15e6-7804-11ef-baf6-efdc52797490@osd.0.service: Consumed 2min 37.613s CPU time. 2024-09-21T10:43:24.798 INFO:journalctl@ceph.osd.0.smithi123.stdout:Sep 21 10:43:24 smithi123 systemd[1]: Starting Ceph osd.0 for 3ccd15e6-7804-11ef-baf6-efdc52797490... 2024-09-21T10:43:24.820 INFO:teuthology.orchestra.run.smithi123.stdout: 5 165286 10.65 MB/sec execute 265 sec latency 59.924 ms 2024-09-21T10:43:25.160 INFO:journalctl@ceph.osd.0.smithi123.stdout:Sep 21 10:43:25 smithi123 podman[68376]: 2024-09-21 10:43:24.923511676 +0000 UTC m=+0.025448808 image pull 8f4d013db793795fef09ff5edf96e6b42ba8cd2c9158b2f0b7a5d7c1e657c063 quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472 2024-09-21T10:43:25.160 INFO:journalctl@ceph.osd.0.smithi123.stdout:Sep 21 10:43:25 smithi123 podman[68376]: 2024-09-21 10:43:25.043041539 +0000 UTC m=+0.144978671 container create e1c6c94f97df8513015ddde1d61e9b2af460973400b0132a886639a2cc1644ca (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-0-activate, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-09-21T10:43:25.412 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:25 smithi123 ceph-mon[63856]: pgmap v56: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 338 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 7.9 MiB/s wr, 808 op/s; 192/1464 objects degraded (13.115%) 2024-09-21T10:43:25.412 INFO:journalctl@ceph.osd.0.smithi123.stdout:Sep 21 10:43:25 smithi123 podman[68376]: 2024-09-21 10:43:25.337959978 +0000 UTC m=+0.439897114 container init e1c6c94f97df8513015ddde1d61e9b2af460973400b0132a886639a2cc1644ca (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-0-activate, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, ceph=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T10:43:25.413 INFO:journalctl@ceph.osd.0.smithi123.stdout:Sep 21 10:43:25 smithi123 podman[68376]: 2024-09-21 10:43:25.350969916 +0000 UTC m=+0.452907049 container start e1c6c94f97df8513015ddde1d61e9b2af460973400b0132a886639a2cc1644ca (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-0-activate, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T10:43:25.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:25 smithi175 ceph-mon[49304]: pgmap v56: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 338 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 7.9 MiB/s wr, 808 op/s; 192/1464 objects degraded (13.115%) 2024-09-21T10:43:25.663 INFO:journalctl@ceph.osd.0.smithi123.stdout:Sep 21 10:43:25 smithi123 podman[68376]: 2024-09-21 10:43:25.490075214 +0000 UTC m=+0.592012344 container attach e1c6c94f97df8513015ddde1d61e9b2af460973400b0132a886639a2cc1644ca (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-0-activate, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T10:43:25.663 INFO:journalctl@ceph.osd.0.smithi123.stdout:Sep 21 10:43:25 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-0-activate[68390]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:43:25.663 INFO:journalctl@ceph.osd.0.smithi123.stdout:Sep 21 10:43:25 smithi123 bash[68376]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:43:25.663 INFO:journalctl@ceph.osd.0.smithi123.stdout:Sep 21 10:43:25 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-0-activate[68390]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:43:25.663 INFO:journalctl@ceph.osd.0.smithi123.stdout:Sep 21 10:43:25 smithi123 bash[68376]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:43:25.820 INFO:teuthology.orchestra.run.smithi123.stdout: 5 165754 10.64 MB/sec execute 266 sec latency 38.436 ms 2024-09-21T10:43:26.821 INFO:teuthology.orchestra.run.smithi123.stdout: 5 166050 10.64 MB/sec execute 267 sec latency 36.436 ms 2024-09-21T10:43:27.449 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:27 smithi123 ceph-mon[63856]: pgmap v57: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 338 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 7.9 MiB/s wr, 808 op/s; 192/1464 objects degraded (13.115%) 2024-09-21T10:43:27.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:27 smithi175 ceph-mon[49304]: pgmap v57: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 338 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 7.9 MiB/s wr, 808 op/s; 192/1464 objects degraded (13.115%) 2024-09-21T10:43:27.821 INFO:teuthology.orchestra.run.smithi123.stdout: 5 166425 10.67 MB/sec execute 268 sec latency 43.750 ms 2024-09-21T10:43:28.732 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:28 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:43:28.733 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:28 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:43:28.821 INFO:teuthology.orchestra.run.smithi123.stdout: 5 166711 10.66 MB/sec execute 269 sec latency 46.232 ms 2024-09-21T10:43:28.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:28 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:43:28.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:28 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:43:29.744 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:29 smithi123 ceph-mon[63856]: pgmap v58: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 338 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 2.2 MiB/s wr, 260 op/s; 192/1464 objects degraded (13.115%) 2024-09-21T10:43:29.821 INFO:teuthology.orchestra.run.smithi123.stdout: 5 167027 10.67 MB/sec execute 270 sec latency 46.323 ms 2024-09-21T10:43:29.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:29 smithi175 ceph-mon[49304]: pgmap v58: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 338 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 2.2 MiB/s wr, 260 op/s; 192/1464 objects degraded (13.115%) 2024-09-21T10:43:30.253 INFO:journalctl@ceph.osd.0.smithi123.stdout:Sep 21 10:43:30 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-0-activate[68390]: --> Failed to activate via raw: did not find any matching OSD to activate 2024-09-21T10:43:30.253 INFO:journalctl@ceph.osd.0.smithi123.stdout:Sep 21 10:43:30 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-0-activate[68390]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:43:30.254 INFO:journalctl@ceph.osd.0.smithi123.stdout:Sep 21 10:43:30 smithi123 bash[68376]: --> Failed to activate via raw: did not find any matching OSD to activate 2024-09-21T10:43:30.254 INFO:journalctl@ceph.osd.0.smithi123.stdout:Sep 21 10:43:30 smithi123 bash[68376]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:43:30.254 INFO:journalctl@ceph.osd.0.smithi123.stdout:Sep 21 10:43:30 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-0-activate[68390]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:43:30.254 INFO:journalctl@ceph.osd.0.smithi123.stdout:Sep 21 10:43:30 smithi123 bash[68376]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:43:30.254 INFO:journalctl@ceph.osd.0.smithi123.stdout:Sep 21 10:43:30 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-0-activate[68390]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-21T10:43:30.254 INFO:journalctl@ceph.osd.0.smithi123.stdout:Sep 21 10:43:30 smithi123 bash[68376]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-21T10:43:30.254 INFO:journalctl@ceph.osd.0.smithi123.stdout:Sep 21 10:43:30 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-0-activate[68390]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-0 --no-mon-config 2024-09-21T10:43:30.254 INFO:journalctl@ceph.osd.0.smithi123.stdout:Sep 21 10:43:30 smithi123 bash[68376]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-0 --no-mon-config 2024-09-21T10:43:30.780 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:30 smithi123 ceph-mon[63856]: Health check update: Degraded data redundancy: 214/1644 objects degraded (13.017%), 29 pgs degraded (PG_DEGRADED) 2024-09-21T10:43:30.780 INFO:journalctl@ceph.osd.0.smithi123.stdout:Sep 21 10:43:30 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-0-activate[68390]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-21T10:43:30.780 INFO:journalctl@ceph.osd.0.smithi123.stdout:Sep 21 10:43:30 smithi123 bash[68376]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-21T10:43:30.781 INFO:journalctl@ceph.osd.0.smithi123.stdout:Sep 21 10:43:30 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-0-activate[68390]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-0/block 2024-09-21T10:43:30.781 INFO:journalctl@ceph.osd.0.smithi123.stdout:Sep 21 10:43:30 smithi123 bash[68376]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-0/block 2024-09-21T10:43:30.781 INFO:journalctl@ceph.osd.0.smithi123.stdout:Sep 21 10:43:30 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-0-activate[68390]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-21T10:43:30.781 INFO:journalctl@ceph.osd.0.smithi123.stdout:Sep 21 10:43:30 smithi123 bash[68376]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-21T10:43:30.781 INFO:journalctl@ceph.osd.0.smithi123.stdout:Sep 21 10:43:30 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-0-activate[68390]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-21T10:43:30.781 INFO:journalctl@ceph.osd.0.smithi123.stdout:Sep 21 10:43:30 smithi123 bash[68376]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-21T10:43:30.781 INFO:journalctl@ceph.osd.0.smithi123.stdout:Sep 21 10:43:30 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-0-activate[68390]: --> ceph-volume lvm activate successful for osd ID: 0 2024-09-21T10:43:30.781 INFO:journalctl@ceph.osd.0.smithi123.stdout:Sep 21 10:43:30 smithi123 bash[68376]: --> ceph-volume lvm activate successful for osd ID: 0 2024-09-21T10:43:30.781 INFO:journalctl@ceph.osd.0.smithi123.stdout:Sep 21 10:43:30 smithi123 podman[68376]: 2024-09-21 10:43:30.611469291 +0000 UTC m=+5.713406414 container died e1c6c94f97df8513015ddde1d61e9b2af460973400b0132a886639a2cc1644ca (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-0-activate, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, GIT_CLEAN=True, ceph=True) 2024-09-21T10:43:30.821 INFO:teuthology.orchestra.run.smithi123.stdout: 5 167256 10.64 MB/sec execute 271 sec latency 73.675 ms 2024-09-21T10:43:31.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:30 smithi175 ceph-mon[49304]: Health check update: Degraded data redundancy: 214/1644 objects degraded (13.017%), 29 pgs degraded (PG_DEGRADED) 2024-09-21T10:43:31.544 INFO:journalctl@ceph.osd.0.smithi123.stdout:Sep 21 10:43:31 smithi123 podman[68614]: 2024-09-21 10:43:31.46280568 +0000 UTC m=+0.840442865 container remove e1c6c94f97df8513015ddde1d61e9b2af460973400b0132a886639a2cc1644ca (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-0-activate, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240731) 2024-09-21T10:43:31.813 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:31 smithi123 ceph-mon[63856]: pgmap v59: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 345 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.5 MiB/s wr, 457 op/s; 214/1644 objects degraded (13.017%) 2024-09-21T10:43:31.813 INFO:journalctl@ceph.osd.0.smithi123.stdout:Sep 21 10:43:31 smithi123 podman[68667]: 2024-09-21 10:43:31.618425455 +0000 UTC m=+0.022304927 image pull 8f4d013db793795fef09ff5edf96e6b42ba8cd2c9158b2f0b7a5d7c1e657c063 quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472 2024-09-21T10:43:31.813 INFO:journalctl@ceph.osd.0.smithi123.stdout:Sep 21 10:43:31 smithi123 podman[68667]: 2024-09-21 10:43:31.753351144 +0000 UTC m=+0.157230601 container create cc40f695c3a88cc6d6448d93b429fc2f60765828ab0657e4d7598ee26bb1d5dd (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD) 2024-09-21T10:43:31.821 INFO:teuthology.orchestra.run.smithi123.stdout: 5 167518 10.62 MB/sec execute 272 sec latency 38.424 ms 2024-09-21T10:43:32.083 INFO:journalctl@ceph.osd.0.smithi123.stdout:Sep 21 10:43:32 smithi123 podman[68667]: 2024-09-21 10:43:32.073165537 +0000 UTC m=+0.477045016 container init cc40f695c3a88cc6d6448d93b429fc2f60765828ab0657e4d7598ee26bb1d5dd (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-0, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True) 2024-09-21T10:43:32.098 INFO:journalctl@ceph.osd.0.smithi123.stdout:Sep 21 10:43:32 smithi123 podman[68667]: 2024-09-21 10:43:32.081349894 +0000 UTC m=+0.485229349 container start cc40f695c3a88cc6d6448d93b429fc2f60765828ab0657e4d7598ee26bb1d5dd (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-0, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-09-21T10:43:32.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:31 smithi175 ceph-mon[49304]: pgmap v59: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 345 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.5 MiB/s wr, 457 op/s; 214/1644 objects degraded (13.017%) 2024-09-21T10:43:32.135 INFO:journalctl@ceph.osd.0.smithi123.stdout:Sep 21 10:43:32 smithi123 ceph-osd[68684]: -- 172.21.15.123:0/2945174467 <== mon.1 v2:172.21.15.175:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x562491dcc1e0 con 0x562490f67c00 2024-09-21T10:43:32.316 INFO:journalctl@ceph.osd.0.smithi123.stdout:Sep 21 10:43:32 smithi123 bash[68667]: cc40f695c3a88cc6d6448d93b429fc2f60765828ab0657e4d7598ee26bb1d5dd 2024-09-21T10:43:32.316 INFO:journalctl@ceph.osd.0.smithi123.stdout:Sep 21 10:43:32 smithi123 systemd[1]: Started Ceph osd.0 for 3ccd15e6-7804-11ef-baf6-efdc52797490. 2024-09-21T10:43:32.821 INFO:teuthology.orchestra.run.smithi123.stdout: 5 167827 10.60 MB/sec execute 273 sec latency 60.162 ms 2024-09-21T10:43:33.387 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:33 smithi123 ceph-mon[63856]: pgmap v60: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 345 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.1 MiB/s wr, 415 op/s; 214/1644 objects degraded (13.017%) 2024-09-21T10:43:33.387 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:33 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:43:33.387 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:33 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:43:33.387 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:33 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:43:33.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:33 smithi175 ceph-mon[49304]: pgmap v60: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 345 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.1 MiB/s wr, 415 op/s; 214/1644 objects degraded (13.017%) 2024-09-21T10:43:33.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:33 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:43:33.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:33 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:43:33.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:33 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:43:33.821 INFO:teuthology.orchestra.run.smithi123.stdout: 5 168084 10.58 MB/sec execute 274 sec latency 71.124 ms 2024-09-21T10:43:34.419 INFO:journalctl@ceph.osd.0.smithi123.stdout:Sep 21 10:43:34 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-0[68680]: 2024-09-21T10:43:34.219+0000 7ff55be23640 -1 Falling back to public interface 2024-09-21T10:43:34.821 INFO:teuthology.orchestra.run.smithi123.stdout: 5 168400 10.56 MB/sec execute 275 sec latency 42.675 ms 2024-09-21T10:43:34.966 INFO:teuthology.orchestra.run.smithi123.stdout:true 2024-09-21T10:43:35.451 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:35 smithi123 ceph-mon[63856]: pgmap v61: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 346 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 5.4 MiB/s wr, 561 op/s; 219/1671 objects degraded (13.106%) 2024-09-21T10:43:35.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:35 smithi175 ceph-mon[49304]: pgmap v61: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 346 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 5.4 MiB/s wr, 561 op/s; 219/1671 objects degraded (13.106%) 2024-09-21T10:43:35.821 INFO:teuthology.orchestra.run.smithi123.stdout: 5 168642 10.56 MB/sec execute 276 sec latency 62.837 ms 2024-09-21T10:43:35.884 INFO:teuthology.orchestra.run.smithi123.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T10:43:35.884 INFO:teuthology.orchestra.run.smithi123.stdout:alertmanager.smithi123 smithi123 *:9093,9094 running (4m) 31s ago 13m 18.0M - 0.27.0 11f11916f8cd 5d550bd97de5 2024-09-21T10:43:35.884 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi123 smithi123 running (47s) 31s ago 13m 6899k - 19.3.0-5103-g5c1cdc3c 8f4d013db793 c5e1ef39a0a8 2024-09-21T10:43:35.884 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi175 smithi175 running (41s) 36s ago 12m 6895k - 19.3.0-5103-g5c1cdc3c 8f4d013db793 fb1ca999ccfd 2024-09-21T10:43:35.884 INFO:teuthology.orchestra.run.smithi123.stdout:grafana.smithi123 smithi123 *:3000 running (3m) 31s ago 12m 43.4M - dad864ee21e9 771df81763ba 2024-09-21T10:43:35.884 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi123.uqoyow smithi123 running (7m) 31s ago 7m 40.9M - 17.2.0 e1d6a67b021e f0a21eaac266 2024-09-21T10:43:35.885 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi175.jrlevw smithi175 running (7m) 36s ago 7m 13.5M - 17.2.0 e1d6a67b021e b21b16f4bc50 2024-09-21T10:43:35.885 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi123.stnuit smithi123 *:8443,9283,8765 running (3m) 31s ago 14m 509M - 19.3.0-5103-g5c1cdc3c 8f4d013db793 6812cc5cb56b 2024-09-21T10:43:35.885 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi175.txvwrq smithi175 *:8443,9283,8765 running (2m) 36s ago 12m 453M - 19.3.0-5103-g5c1cdc3c 8f4d013db793 304f0408075e 2024-09-21T10:43:35.885 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi123 smithi123 running (2m) 31s ago 14m 43.6M 2048M 19.3.0-5103-g5c1cdc3c 8f4d013db793 6b95207a2d80 2024-09-21T10:43:35.885 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi175 smithi175 running (97s) 36s ago 12m 36.7M 2048M 19.3.0-5103-g5c1cdc3c 8f4d013db793 3abee2d37f0f 2024-09-21T10:43:35.885 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.0.0.smithi123.ddatpr smithi123 *:2049 running (6m) 31s ago 6m 282M - 4.0 e1d6a67b021e 36f8cb2281eb 2024-09-21T10:43:35.885 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.1.0.smithi175.wpawer smithi175 *:2049 running (6m) 36s ago 6m 90.3M - 4.0 e1d6a67b021e 2200aeed9200 2024-09-21T10:43:35.885 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi123 smithi123 *:9100 running (3m) 31s ago 12m 10.4M - 1.7.0 72c9c2088986 ca7d0b29af81 2024-09-21T10:43:35.885 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi175 smithi175 *:9100 running (3m) 36s ago 12m 11.1M - 1.7.0 72c9c2088986 46c4f6b7c149 2024-09-21T10:43:35.885 INFO:teuthology.orchestra.run.smithi123.stdout:osd.0 smithi123 starting - - - 1936M 2024-09-21T10:43:35.885 INFO:teuthology.orchestra.run.smithi123.stdout:osd.1 smithi123 running (11m) 31s ago 11m 475M 1936M 17.2.0 e1d6a67b021e 59aee515b0f8 2024-09-21T10:43:35.885 INFO:teuthology.orchestra.run.smithi123.stdout:osd.2 smithi123 running (10m) 31s ago 10m 367M 1936M 17.2.0 e1d6a67b021e 7db93f8ce5b5 2024-09-21T10:43:35.885 INFO:teuthology.orchestra.run.smithi123.stdout:osd.3 smithi123 running (10m) 31s ago 10m 490M 1936M 17.2.0 e1d6a67b021e c1672dea7b1f 2024-09-21T10:43:35.886 INFO:teuthology.orchestra.run.smithi123.stdout:osd.4 smithi175 running (9m) 36s ago 9m 498M 2704M 17.2.0 e1d6a67b021e 48fe728c259e 2024-09-21T10:43:35.886 INFO:teuthology.orchestra.run.smithi123.stdout:osd.5 smithi175 running (9m) 36s ago 9m 331M 2704M 17.2.0 e1d6a67b021e e270d5a21b14 2024-09-21T10:43:35.886 INFO:teuthology.orchestra.run.smithi123.stdout:osd.6 smithi175 running (9m) 36s ago 9m 517M 2704M 17.2.0 e1d6a67b021e 9bcbf018eb20 2024-09-21T10:43:35.886 INFO:teuthology.orchestra.run.smithi123.stdout:osd.7 smithi175 running (8m) 36s ago 8m 383M 2704M 17.2.0 e1d6a67b021e e9b8f1f90994 2024-09-21T10:43:35.886 INFO:teuthology.orchestra.run.smithi123.stdout:prometheus.smithi123 smithi123 *:9095 running (2m) 31s ago 12m 48.4M - 2.51.0 1d3b7f56885b 37f944c9de88 2024-09-21T10:43:36.502 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-09-21T10:43:36.502 INFO:teuthology.orchestra.run.smithi123.stdout: "mon": { 2024-09-21T10:43:36.502 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev)": 2 2024-09-21T10:43:36.502 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:43:36.502 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr": { 2024-09-21T10:43:36.502 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev)": 2 2024-09-21T10:43:36.502 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:43:36.502 INFO:teuthology.orchestra.run.smithi123.stdout: "osd": { 2024-09-21T10:43:36.502 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7 2024-09-21T10:43:36.503 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:43:36.503 INFO:teuthology.orchestra.run.smithi123.stdout: "mds": { 2024-09-21T10:43:36.503 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T10:43:36.503 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:43:36.503 INFO:teuthology.orchestra.run.smithi123.stdout: "overall": { 2024-09-21T10:43:36.503 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 9, 2024-09-21T10:43:36.503 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev)": 4 2024-09-21T10:43:36.503 INFO:teuthology.orchestra.run.smithi123.stdout: } 2024-09-21T10:43:36.503 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-09-21T10:43:36.504 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:36 smithi123 ceph-mon[63856]: from='client.44151 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:43:36.504 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:36 smithi123 ceph-mon[63856]: from='client.44155 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:43:36.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:36 smithi175 ceph-mon[49304]: from='client.44151 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:43:36.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:36 smithi175 ceph-mon[49304]: from='client.44155 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:43:36.821 INFO:teuthology.orchestra.run.smithi123.stdout: 5 168981 10.54 MB/sec execute 277 sec latency 90.871 ms 2024-09-21T10:43:37.007 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-09-21T10:43:37.007 INFO:teuthology.orchestra.run.smithi123.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472", 2024-09-21T10:43:37.007 INFO:teuthology.orchestra.run.smithi123.stdout: "in_progress": true, 2024-09-21T10:43:37.007 INFO:teuthology.orchestra.run.smithi123.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T10:43:37.008 INFO:teuthology.orchestra.run.smithi123.stdout: "services_complete": [ 2024-09-21T10:43:37.008 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr", 2024-09-21T10:43:37.008 INFO:teuthology.orchestra.run.smithi123.stdout: "crash", 2024-09-21T10:43:37.008 INFO:teuthology.orchestra.run.smithi123.stdout: "mon" 2024-09-21T10:43:37.008 INFO:teuthology.orchestra.run.smithi123.stdout: ], 2024-09-21T10:43:37.008 INFO:teuthology.orchestra.run.smithi123.stdout: "progress": "6/23 daemons upgraded", 2024-09-21T10:43:37.008 INFO:teuthology.orchestra.run.smithi123.stdout: "message": "Currently upgrading osd daemons", 2024-09-21T10:43:37.008 INFO:teuthology.orchestra.run.smithi123.stdout: "is_paused": false 2024-09-21T10:43:37.008 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-09-21T10:43:37.562 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:37 smithi123 ceph-mon[63856]: from='client.34208 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:43:37.562 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:37 smithi123 ceph-mon[63856]: pgmap v62: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 346 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 3.9 MiB/s wr, 387 op/s; 219/1671 objects degraded (13.106%) 2024-09-21T10:43:37.562 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:37 smithi123 ceph-mon[63856]: from='client.? 172.21.15.123:0/308560953' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:43:37.591 INFO:teuthology.orchestra.run.smithi123.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 219/1671 objects degraded (13.106%), 29 pgs degraded 2024-09-21T10:43:37.592 INFO:teuthology.orchestra.run.smithi123.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-21T10:43:37.592 INFO:teuthology.orchestra.run.smithi123.stdout: osd.0 (root=default,host=smithi123) is down 2024-09-21T10:43:37.592 INFO:teuthology.orchestra.run.smithi123.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 219/1671 objects degraded (13.106%), 29 pgs degraded 2024-09-21T10:43:37.592 INFO:teuthology.orchestra.run.smithi123.stdout: pg 1.0 is active+undersized+degraded, acting [7,6] 2024-09-21T10:43:37.592 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.0 is active+undersized+degraded, acting [2,1] 2024-09-21T10:43:37.592 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.1 is active+undersized+degraded, acting [2,3] 2024-09-21T10:43:37.592 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.4 is active+undersized+degraded, acting [1,7] 2024-09-21T10:43:37.592 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.5 is active+undersized+degraded, acting [7,4] 2024-09-21T10:43:37.592 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.8 is active+undersized+degraded, acting [7,1] 2024-09-21T10:43:37.592 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.c is active+undersized+degraded, acting [2,5] 2024-09-21T10:43:37.592 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.f is active+undersized+degraded, acting [4,7] 2024-09-21T10:43:37.592 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.10 is active+undersized+degraded, acting [2,1] 2024-09-21T10:43:37.592 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.13 is active+undersized+degraded, acting [4,3] 2024-09-21T10:43:37.592 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.15 is active+undersized+degraded, acting [1,5] 2024-09-21T10:43:37.593 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.19 is active+undersized+degraded, acting [3,6] 2024-09-21T10:43:37.593 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.1d is active+undersized+degraded, acting [7,6] 2024-09-21T10:43:37.593 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.1e is active+undersized+degraded, acting [3,5] 2024-09-21T10:43:37.593 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.1f is active+undersized+degraded, acting [7,4] 2024-09-21T10:43:37.593 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.1 is active+undersized+degraded, acting [4,3] 2024-09-21T10:43:37.593 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.3 is active+undersized+degraded, acting [4,6] 2024-09-21T10:43:37.593 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.6 is active+undersized+degraded, acting [1,4] 2024-09-21T10:43:37.593 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.7 is active+undersized+degraded, acting [3,7] 2024-09-21T10:43:37.593 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.b is active+undersized+degraded, acting [3,4] 2024-09-21T10:43:37.593 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.f is active+undersized+degraded, acting [7,4] 2024-09-21T10:43:37.593 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.10 is active+undersized+degraded, acting [6,5] 2024-09-21T10:43:37.593 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.12 is active+undersized+degraded, acting [7,3] 2024-09-21T10:43:37.593 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.17 is active+undersized+degraded, acting [5,3] 2024-09-21T10:43:37.593 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.18 is active+undersized+degraded, acting [3,1] 2024-09-21T10:43:37.593 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.1b is active+undersized+degraded, acting [4,7] 2024-09-21T10:43:37.594 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.1f is active+undersized+degraded, acting [5,2] 2024-09-21T10:43:37.594 INFO:teuthology.orchestra.run.smithi123.stdout: pg 4.0 is active+undersized+degraded, acting [3,7] 2024-09-21T10:43:37.594 INFO:teuthology.orchestra.run.smithi123.stdout: pg 4.19 is active+undersized+degraded, acting [3,2] 2024-09-21T10:43:37.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:37 smithi175 ceph-mon[49304]: from='client.34208 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:43:37.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:37 smithi175 ceph-mon[49304]: pgmap v62: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 346 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 3.9 MiB/s wr, 387 op/s; 219/1671 objects degraded (13.106%) 2024-09-21T10:43:37.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:37 smithi175 ceph-mon[49304]: from='client.? 172.21.15.123:0/308560953' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:43:37.821 INFO:teuthology.orchestra.run.smithi123.stdout: 5 169353 10.53 MB/sec execute 278 sec latency 47.149 ms 2024-09-21T10:43:38.570 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:38 smithi123 ceph-mon[63856]: from='client.44165 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:43:38.570 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:38 smithi123 ceph-mon[63856]: from='client.? 172.21.15.123:0/2045652470' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T10:43:38.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:38 smithi175 ceph-mon[49304]: from='client.44165 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:43:38.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:38 smithi175 ceph-mon[49304]: from='client.? 172.21.15.123:0/2045652470' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T10:43:38.821 INFO:teuthology.orchestra.run.smithi123.stdout: 5 169786 10.56 MB/sec execute 279 sec latency 34.216 ms 2024-09-21T10:43:39.591 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:39 smithi123 ceph-mon[63856]: pgmap v63: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 346 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 3.9 MiB/s wr, 387 op/s; 219/1671 objects degraded (13.106%) 2024-09-21T10:43:39.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:39 smithi175 ceph-mon[49304]: pgmap v63: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 346 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 3.9 MiB/s wr, 387 op/s; 219/1671 objects degraded (13.106%) 2024-09-21T10:43:39.822 INFO:teuthology.orchestra.run.smithi123.stdout: 5 170253 10.58 MB/sec execute 280 sec latency 32.562 ms 2024-09-21T10:43:40.616 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:40 smithi123 ceph-mon[63856]: Health check update: Degraded data redundancy: 219/1671 objects degraded (13.106%), 29 pgs degraded (PG_DEGRADED) 2024-09-21T10:43:40.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:40 smithi175 ceph-mon[49304]: Health check update: Degraded data redundancy: 219/1671 objects degraded (13.106%), 29 pgs degraded (PG_DEGRADED) 2024-09-21T10:43:40.822 INFO:teuthology.orchestra.run.smithi123.stdout: 5 170594 10.59 MB/sec execute 281 sec latency 44.877 ms 2024-09-21T10:43:41.620 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:41 smithi123 ceph-mon[63856]: pgmap v64: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 343 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.7 MiB/s wr, 577 op/s; 210/1605 objects degraded (13.084%) 2024-09-21T10:43:41.620 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:41 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:43:41.620 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:41 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:43:41.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:41 smithi175 ceph-mon[49304]: pgmap v64: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 343 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.7 MiB/s wr, 577 op/s; 210/1605 objects degraded (13.084%) 2024-09-21T10:43:41.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:41 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:43:41.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:41 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:43:41.822 INFO:teuthology.orchestra.run.smithi123.stdout: 5 170773 10.56 MB/sec execute 282 sec latency 52.424 ms 2024-09-21T10:43:42.822 INFO:teuthology.orchestra.run.smithi123.stdout: 5 170936 10.54 MB/sec execute 283 sec latency 69.175 ms 2024-09-21T10:43:42.885 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:42 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:43:42.885 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:42 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:43:42.885 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:42 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:43:43.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:42 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:43:43.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:42 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:43:43.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:42 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:43:43.136 INFO:journalctl@ceph.osd.0.smithi123.stdout:Sep 21 10:43:43 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-0[68680]: 2024-09-21T10:43:43.019+0000 7ff55be23640 -1 osd.0 0 read_superblock omap replica is missing. 2024-09-21T10:43:43.822 INFO:teuthology.orchestra.run.smithi123.stdout: 5 171191 10.52 MB/sec execute 284 sec latency 88.135 ms 2024-09-21T10:43:43.896 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:43 smithi123 ceph-mon[63856]: pgmap v65: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 343 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.5 MiB/s wr, 372 op/s; 210/1605 objects degraded (13.084%) 2024-09-21T10:43:44.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:43 smithi175 ceph-mon[49304]: pgmap v65: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 343 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.5 MiB/s wr, 372 op/s; 210/1605 objects degraded (13.084%) 2024-09-21T10:43:44.147 INFO:journalctl@ceph.osd.0.smithi123.stdout:Sep 21 10:43:44 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-0[68680]: 2024-09-21T10:43:44.142+0000 7ff55be23640 -1 osd.0 76 log_to_monitors true 2024-09-21T10:43:44.822 INFO:teuthology.orchestra.run.smithi123.stdout: 5 171489 10.50 MB/sec execute 285 sec latency 88.409 ms 2024-09-21T10:43:44.993 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:44 smithi123 ceph-mon[63856]: from='osd.0 [v2:172.21.15.123:6802/3197472985,v1:172.21.15.123:6803/3197472985]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-21T10:43:45.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:44 smithi175 ceph-mon[49304]: from='osd.0 [v2:172.21.15.123:6802/3197472985,v1:172.21.15.123:6803/3197472985]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-21T10:43:45.513 INFO:journalctl@ceph.osd.0.smithi123.stdout:Sep 21 10:43:45 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-0[68680]: 2024-09-21T10:43:45.297+0000 7ff54fbc5640 -1 osd.0 76 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T10:43:45.764 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:45 smithi123 ceph-mon[63856]: pgmap v66: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 344 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.7 MiB/s wr, 561 op/s; 215/1572 objects degraded (13.677%) 2024-09-21T10:43:45.822 INFO:teuthology.orchestra.run.smithi123.stdout: 5 171743 10.47 MB/sec execute 286 sec latency 195.077 ms 2024-09-21T10:43:46.015 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:45 smithi123 ceph-mon[63856]: from='osd.0 [v2:172.21.15.123:6802/3197472985,v1:172.21.15.123:6803/3197472985]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-21T10:43:46.015 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:45 smithi123 ceph-mon[63856]: osdmap e79: 8 total, 7 up, 8 in 2024-09-21T10:43:46.015 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:45 smithi123 ceph-mon[63856]: from='osd.0 [v2:172.21.15.123:6802/3197472985,v1:172.21.15.123:6803/3197472985]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi123", "root=default"]} : dispatch 2024-09-21T10:43:46.015 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:45 smithi123 ceph-mon[63856]: Health check update: Degraded data redundancy: 215/1572 objects degraded (13.677%), 29 pgs degraded (PG_DEGRADED) 2024-09-21T10:43:46.015 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:45 smithi123 ceph-mon[63856]: from='osd.0 [v2:172.21.15.123:6802/3197472985,v1:172.21.15.123:6803/3197472985]' entity='osd.0' 2024-09-21T10:43:46.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:45 smithi175 ceph-mon[49304]: pgmap v66: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 344 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.7 MiB/s wr, 561 op/s; 215/1572 objects degraded (13.677%) 2024-09-21T10:43:46.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:45 smithi175 ceph-mon[49304]: from='osd.0 [v2:172.21.15.123:6802/3197472985,v1:172.21.15.123:6803/3197472985]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-21T10:43:46.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:45 smithi175 ceph-mon[49304]: osdmap e79: 8 total, 7 up, 8 in 2024-09-21T10:43:46.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:45 smithi175 ceph-mon[49304]: from='osd.0 [v2:172.21.15.123:6802/3197472985,v1:172.21.15.123:6803/3197472985]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi123", "root=default"]} : dispatch 2024-09-21T10:43:46.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:45 smithi175 ceph-mon[49304]: Health check update: Degraded data redundancy: 215/1572 objects degraded (13.677%), 29 pgs degraded (PG_DEGRADED) 2024-09-21T10:43:46.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:45 smithi175 ceph-mon[49304]: from='osd.0 [v2:172.21.15.123:6802/3197472985,v1:172.21.15.123:6803/3197472985]' entity='osd.0' 2024-09-21T10:43:46.822 INFO:teuthology.orchestra.run.smithi123.stdout: 5 171912 10.45 MB/sec execute 287 sec latency 561.296 ms 2024-09-21T10:43:47.029 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:46 smithi123 ceph-mon[63856]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T10:43:47.030 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:46 smithi123 ceph-mon[63856]: osd.0 [v2:172.21.15.123:6802/3197472985,v1:172.21.15.123:6803/3197472985] boot 2024-09-21T10:43:47.030 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:46 smithi123 ceph-mon[63856]: osdmap e80: 8 total, 8 up, 8 in 2024-09-21T10:43:47.030 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:46 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-21T10:43:47.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:46 smithi175 ceph-mon[49304]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T10:43:47.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:46 smithi175 ceph-mon[49304]: osd.0 [v2:172.21.15.123:6802/3197472985,v1:172.21.15.123:6803/3197472985] boot 2024-09-21T10:43:47.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:46 smithi175 ceph-mon[49304]: osdmap e80: 8 total, 8 up, 8 in 2024-09-21T10:43:47.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:46 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-21T10:43:47.823 INFO:teuthology.orchestra.run.smithi123.stdout: 5 171912 10.42 MB/sec execute 288 sec latency 1561.348 ms 2024-09-21T10:43:48.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:47 smithi123 ceph-mon[63856]: pgmap v68: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.8 MiB/s wr, 454 op/s; 215/1572 objects degraded (13.677%) 2024-09-21T10:43:48.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:47 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:43:48.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:47 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:43:48.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:47 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:43:48.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:47 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:43:48.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:47 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:43:48.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:47 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:43:48.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:47 smithi123 ceph-mon[63856]: osdmap e81: 8 total, 8 up, 8 in 2024-09-21T10:43:48.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:47 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:43:48.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:47 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:43:48.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:47 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:43:48.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:47 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:43:48.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:47 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-21T10:43:48.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:47 smithi175 ceph-mon[49304]: pgmap v68: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.8 MiB/s wr, 454 op/s; 215/1572 objects degraded (13.677%) 2024-09-21T10:43:48.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:47 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:43:48.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:47 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:43:48.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:47 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:43:48.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:47 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:43:48.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:47 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:43:48.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:47 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:43:48.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:47 smithi175 ceph-mon[49304]: osdmap e81: 8 total, 8 up, 8 in 2024-09-21T10:43:48.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:47 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:43:48.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:47 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:43:48.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:47 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:43:48.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:47 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:43:48.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:47 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-21T10:43:48.823 INFO:teuthology.orchestra.run.smithi123.stdout: 5 172030 10.39 MB/sec execute 289 sec latency 2177.332 ms 2024-09-21T10:43:48.943 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:48 smithi123 ceph-mon[63856]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-21T10:43:48.943 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:48 smithi123 ceph-mon[63856]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-09-21T10:43:48.943 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:48 smithi123 ceph-mon[63856]: osdmap e82: 8 total, 8 up, 8 in 2024-09-21T10:43:49.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:48 smithi175 ceph-mon[49304]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-21T10:43:49.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:48 smithi175 ceph-mon[49304]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-09-21T10:43:49.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:48 smithi175 ceph-mon[49304]: osdmap e82: 8 total, 8 up, 8 in 2024-09-21T10:43:49.823 INFO:teuthology.orchestra.run.smithi123.stdout: 5 172155 10.37 MB/sec execute 290 sec latency 566.179 ms 2024-09-21T10:43:50.032 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:49 smithi123 ceph-mon[63856]: pgmap v71: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.4 MiB/s wr, 377 op/s; 215/1572 objects degraded (13.677%) 2024-09-21T10:43:50.032 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:49 smithi123 ceph-mon[63856]: osdmap e83: 8 total, 8 up, 8 in 2024-09-21T10:43:50.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:49 smithi175 ceph-mon[49304]: pgmap v71: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.4 MiB/s wr, 377 op/s; 215/1572 objects degraded (13.677%) 2024-09-21T10:43:50.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:49 smithi175 ceph-mon[49304]: osdmap e83: 8 total, 8 up, 8 in 2024-09-21T10:43:50.823 INFO:teuthology.orchestra.run.smithi123.stdout: 5 172387 10.35 MB/sec execute 291 sec latency 766.374 ms 2024-09-21T10:43:51.055 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:50 smithi123 ceph-mon[63856]: osdmap e84: 8 total, 8 up, 8 in 2024-09-21T10:43:51.055 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:50 smithi123 ceph-mon[63856]: pgmap v75: 97 pgs: 15 remapped+peering, 82 active+clean; 335 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.1 KiB/s rd, 2.6 MiB/s wr, 348 op/s; 22 B/s, 0 objects/s recovering 2024-09-21T10:43:51.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:50 smithi175 ceph-mon[49304]: osdmap e84: 8 total, 8 up, 8 in 2024-09-21T10:43:51.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:50 smithi175 ceph-mon[49304]: pgmap v75: 97 pgs: 15 remapped+peering, 82 active+clean; 335 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.1 KiB/s rd, 2.6 MiB/s wr, 348 op/s; 22 B/s, 0 objects/s recovering 2024-09-21T10:43:51.823 INFO:teuthology.orchestra.run.smithi123.stdout: 5 172580 10.33 MB/sec execute 292 sec latency 931.937 ms 2024-09-21T10:43:52.066 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:51 smithi123 ceph-mon[63856]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 215/1572 objects degraded (13.677%), 29 pgs degraded) 2024-09-21T10:43:52.066 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:51 smithi123 ceph-mon[63856]: Cluster is now healthy 2024-09-21T10:43:52.066 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:51 smithi123 ceph-mon[63856]: osdmap e85: 8 total, 8 up, 8 in 2024-09-21T10:43:52.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:51 smithi175 ceph-mon[49304]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 215/1572 objects degraded (13.677%), 29 pgs degraded) 2024-09-21T10:43:52.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:51 smithi175 ceph-mon[49304]: Cluster is now healthy 2024-09-21T10:43:52.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:51 smithi175 ceph-mon[49304]: osdmap e85: 8 total, 8 up, 8 in 2024-09-21T10:43:52.823 INFO:teuthology.orchestra.run.smithi123.stdout: 5 172991 10.34 MB/sec execute 293 sec latency 1012.478 ms 2024-09-21T10:43:53.074 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:53 smithi123 ceph-mon[63856]: osdmap e86: 8 total, 8 up, 8 in 2024-09-21T10:43:53.074 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:53 smithi123 ceph-mon[63856]: pgmap v78: 97 pgs: 15 remapped+peering, 82 active+clean; 335 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 111 KiB/s wr, 13 op/s 2024-09-21T10:43:53.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:53 smithi175 ceph-mon[49304]: osdmap e86: 8 total, 8 up, 8 in 2024-09-21T10:43:53.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:53 smithi175 ceph-mon[49304]: pgmap v78: 97 pgs: 15 remapped+peering, 82 active+clean; 335 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 111 KiB/s wr, 13 op/s 2024-09-21T10:43:53.823 INFO:teuthology.orchestra.run.smithi123.stdout: 5 173278 10.34 MB/sec execute 294 sec latency 554.572 ms 2024-09-21T10:43:54.107 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:54 smithi123 ceph-mon[63856]: osdmap e87: 8 total, 8 up, 8 in 2024-09-21T10:43:54.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:54 smithi175 ceph-mon[49304]: osdmap e87: 8 total, 8 up, 8 in 2024-09-21T10:43:54.823 INFO:teuthology.orchestra.run.smithi123.stdout: 5 173622 10.35 MB/sec execute 295 sec latency 769.088 ms 2024-09-21T10:43:55.113 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:55 smithi123 ceph-mon[63856]: osdmap e88: 8 total, 8 up, 8 in 2024-09-21T10:43:55.113 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:55 smithi123 ceph-mon[63856]: pgmap v81: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 4 remapped+peering, 88 active+clean; 340 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 3.7 MiB/s wr, 400 op/s; 94/1557 objects degraded (6.037%); 2.7 MiB/s, 24 objects/s recovering 2024-09-21T10:43:55.113 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:55 smithi123 ceph-mon[63856]: osdmap e89: 8 total, 8 up, 8 in 2024-09-21T10:43:55.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:55 smithi175 ceph-mon[49304]: osdmap e88: 8 total, 8 up, 8 in 2024-09-21T10:43:55.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:55 smithi175 ceph-mon[49304]: pgmap v81: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 4 remapped+peering, 88 active+clean; 340 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 3.7 MiB/s wr, 400 op/s; 94/1557 objects degraded (6.037%); 2.7 MiB/s, 24 objects/s recovering 2024-09-21T10:43:55.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:55 smithi175 ceph-mon[49304]: osdmap e89: 8 total, 8 up, 8 in 2024-09-21T10:43:55.823 INFO:teuthology.orchestra.run.smithi123.stdout: 5 174119 10.35 MB/sec execute 296 sec latency 704.743 ms 2024-09-21T10:43:56.128 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:56 smithi123 ceph-mon[63856]: Health check failed: Degraded data redundancy: 94/1557 objects degraded (6.037%), 5 pgs degraded (PG_DEGRADED) 2024-09-21T10:43:56.128 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:56 smithi123 ceph-mon[63856]: osdmap e90: 8 total, 8 up, 8 in 2024-09-21T10:43:56.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:56 smithi175 ceph-mon[49304]: Health check failed: Degraded data redundancy: 94/1557 objects degraded (6.037%), 5 pgs degraded (PG_DEGRADED) 2024-09-21T10:43:56.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:56 smithi175 ceph-mon[49304]: osdmap e90: 8 total, 8 up, 8 in 2024-09-21T10:43:56.823 INFO:teuthology.orchestra.run.smithi123.stdout: 5 174628 10.37 MB/sec execute 297 sec latency 707.026 ms 2024-09-21T10:43:57.147 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:57 smithi123 ceph-mon[63856]: pgmap v84: 97 pgs: 2 peering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 341 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 4.5 MiB/s wr, 432 op/s; 52/1599 objects degraded (3.252%); 2.7 MiB/s, 65 objects/s recovering 2024-09-21T10:43:57.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:57 smithi175 ceph-mon[49304]: pgmap v84: 97 pgs: 2 peering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 341 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 4.5 MiB/s wr, 432 op/s; 52/1599 objects degraded (3.252%); 2.7 MiB/s, 65 objects/s recovering 2024-09-21T10:43:57.823 INFO:teuthology.orchestra.run.smithi123.stdout: 5 175048 10.36 MB/sec execute 298 sec latency 43.734 ms 2024-09-21T10:43:58.662 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:58 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:43:58.662 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:58 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:43:58.823 INFO:teuthology.orchestra.run.smithi123.stdout: 5 175569 10.37 MB/sec execute 299 sec latency 30.262 ms 2024-09-21T10:43:58.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:58 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:43:58.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:58 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:43:59.673 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:43:59 smithi123 ceph-mon[63856]: pgmap v85: 97 pgs: 2 peering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 341 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.8 KiB/s rd, 3.5 MiB/s wr, 331 op/s; 52/1599 objects degraded (3.252%); 2.1 MiB/s, 50 objects/s recovering 2024-09-21T10:43:59.830 INFO:teuthology.orchestra.run.smithi123.stdout: 5 176034 10.38 MB/sec execute 300 sec latency 34.076 ms 2024-09-21T10:43:59.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:43:59 smithi175 ceph-mon[49304]: pgmap v85: 97 pgs: 2 peering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 341 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.8 KiB/s rd, 3.5 MiB/s wr, 331 op/s; 52/1599 objects degraded (3.252%); 2.1 MiB/s, 50 objects/s recovering 2024-09-21T10:44:00.686 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:00 smithi123 ceph-mon[63856]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 52/1599 objects degraded (3.252%), 3 pgs degraded) 2024-09-21T10:44:00.686 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:00 smithi123 ceph-mon[63856]: Cluster is now healthy 2024-09-21T10:44:00.830 INFO:teuthology.orchestra.run.smithi123.stdout: 5 176394 10.38 MB/sec execute 301 sec latency 50.211 ms 2024-09-21T10:44:01.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:00 smithi175 ceph-mon[49304]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 52/1599 objects degraded (3.252%), 3 pgs degraded) 2024-09-21T10:44:01.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:00 smithi175 ceph-mon[49304]: Cluster is now healthy 2024-09-21T10:44:01.700 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:01 smithi123 ceph-mon[63856]: pgmap v86: 97 pgs: 97 active+clean; 344 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 5.9 MiB/s wr, 533 op/s; 0 B/s, 35 objects/s recovering 2024-09-21T10:44:01.830 INFO:teuthology.orchestra.run.smithi123.stdout: 5 176864 10.38 MB/sec execute 302 sec latency 50.721 ms 2024-09-21T10:44:02.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:01 smithi175 ceph-mon[49304]: pgmap v86: 97 pgs: 97 active+clean; 344 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 5.9 MiB/s wr, 533 op/s; 0 B/s, 35 objects/s recovering 2024-09-21T10:44:02.725 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:02 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-21T10:44:02.831 INFO:teuthology.orchestra.run.smithi123.stdout: 5 177158 10.38 MB/sec execute 303 sec latency 86.853 ms 2024-09-21T10:44:03.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:02 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-21T10:44:03.735 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:03 smithi123 ceph-mon[63856]: pgmap v87: 97 pgs: 97 active+clean; 344 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 4.6 MiB/s wr, 414 op/s; 0 B/s, 27 objects/s recovering 2024-09-21T10:44:03.735 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:03 smithi123 ceph-mon[63856]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-21T10:44:03.735 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:03 smithi123 ceph-mon[63856]: Upgrade: osd.1 is safe to restart 2024-09-21T10:44:03.830 INFO:teuthology.orchestra.run.smithi123.stdout: 5 177507 10.38 MB/sec execute 304 sec latency 50.911 ms 2024-09-21T10:44:04.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:03 smithi175 ceph-mon[49304]: pgmap v87: 97 pgs: 97 active+clean; 344 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 4.6 MiB/s wr, 414 op/s; 0 B/s, 27 objects/s recovering 2024-09-21T10:44:04.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:03 smithi175 ceph-mon[49304]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-21T10:44:04.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:03 smithi175 ceph-mon[49304]: Upgrade: osd.1 is safe to restart 2024-09-21T10:44:04.830 INFO:teuthology.orchestra.run.smithi123.stdout: 5 177848 10.38 MB/sec execute 305 sec latency 85.379 ms 2024-09-21T10:44:05.511 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:05 smithi123 ceph-mon[63856]: pgmap v88: 97 pgs: 97 active+clean; 347 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.4 KiB/s rd, 6.4 MiB/s wr, 641 op/s; 0 B/s, 6 objects/s recovering 2024-09-21T10:44:05.511 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:05 smithi123 ceph-mon[63856]: Upgrade: Updating osd.1 2024-09-21T10:44:05.511 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:05 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:44:05.511 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:05 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-21T10:44:05.512 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:05 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:44:05.512 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:05 smithi123 ceph-mon[63856]: Deploying daemon osd.1 on smithi123 2024-09-21T10:44:05.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:05 smithi175 ceph-mon[49304]: pgmap v88: 97 pgs: 97 active+clean; 347 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.4 KiB/s rd, 6.4 MiB/s wr, 641 op/s; 0 B/s, 6 objects/s recovering 2024-09-21T10:44:05.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:05 smithi175 ceph-mon[49304]: Upgrade: Updating osd.1 2024-09-21T10:44:05.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:05 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:44:05.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:05 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-21T10:44:05.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:05 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:44:05.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:05 smithi175 ceph-mon[49304]: Deploying daemon osd.1 on smithi123 2024-09-21T10:44:05.831 INFO:teuthology.orchestra.run.smithi123.stdout: 5 178230 10.38 MB/sec execute 306 sec latency 33.367 ms 2024-09-21T10:44:06.831 INFO:teuthology.orchestra.run.smithi123.stdout: 5 178436 10.37 MB/sec execute 307 sec latency 96.952 ms 2024-09-21T10:44:07.391 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:07 smithi123 ceph-mon[63856]: pgmap v89: 97 pgs: 97 active+clean; 347 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 6.1 MiB/s wr, 594 op/s; 0 B/s, 5 objects/s recovering 2024-09-21T10:44:07.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:07 smithi175 ceph-mon[49304]: pgmap v89: 97 pgs: 97 active+clean; 347 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 6.1 MiB/s wr, 594 op/s; 0 B/s, 5 objects/s recovering 2024-09-21T10:44:07.831 INFO:teuthology.orchestra.run.smithi123.stdout: 5 178624 10.34 MB/sec execute 308 sec latency 74.268 ms 2024-09-21T10:44:08.082 INFO:teuthology.orchestra.run.smithi123.stdout:true 2024-09-21T10:44:08.679 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:08 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:44:08.831 INFO:teuthology.orchestra.run.smithi123.stdout: 5 178942 10.33 MB/sec execute 309 sec latency 79.566 ms 2024-09-21T10:44:08.996 INFO:teuthology.orchestra.run.smithi123.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T10:44:08.996 INFO:teuthology.orchestra.run.smithi123.stdout:alertmanager.smithi123 smithi123 *:9093,9094 running (4m) 28s ago 13m 18.0M - 0.27.0 11f11916f8cd 5d550bd97de5 2024-09-21T10:44:08.996 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi123 smithi123 running (80s) 28s ago 13m 6899k - 19.3.0-5103-g5c1cdc3c 8f4d013db793 c5e1ef39a0a8 2024-09-21T10:44:08.996 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi175 smithi175 running (74s) 69s ago 13m 6895k - 19.3.0-5103-g5c1cdc3c 8f4d013db793 fb1ca999ccfd 2024-09-21T10:44:08.996 INFO:teuthology.orchestra.run.smithi123.stdout:grafana.smithi123 smithi123 *:3000 running (4m) 28s ago 13m 43.1M - dad864ee21e9 771df81763ba 2024-09-21T10:44:08.997 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi123.uqoyow smithi123 running (7m) 28s ago 7m 40.9M - 17.2.0 e1d6a67b021e f0a21eaac266 2024-09-21T10:44:08.997 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi175.jrlevw smithi175 running (7m) 69s ago 7m 13.5M - 17.2.0 e1d6a67b021e b21b16f4bc50 2024-09-21T10:44:08.997 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi123.stnuit smithi123 *:8443,9283,8765 running (3m) 28s ago 15m 511M - 19.3.0-5103-g5c1cdc3c 8f4d013db793 6812cc5cb56b 2024-09-21T10:44:08.997 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi175.txvwrq smithi175 *:8443,9283,8765 running (2m) 69s ago 13m 453M - 19.3.0-5103-g5c1cdc3c 8f4d013db793 304f0408075e 2024-09-21T10:44:08.997 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi123 smithi123 running (2m) 28s ago 15m 46.5M 2048M 19.3.0-5103-g5c1cdc3c 8f4d013db793 6b95207a2d80 2024-09-21T10:44:08.997 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi175 smithi175 running (2m) 69s ago 13m 36.7M 2048M 19.3.0-5103-g5c1cdc3c 8f4d013db793 3abee2d37f0f 2024-09-21T10:44:08.997 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.0.0.smithi123.ddatpr smithi123 *:2049 running (7m) 28s ago 7m 287M - 4.0 e1d6a67b021e 36f8cb2281eb 2024-09-21T10:44:08.997 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.1.0.smithi175.wpawer smithi175 *:2049 running (7m) 69s ago 7m 90.3M - 4.0 e1d6a67b021e 2200aeed9200 2024-09-21T10:44:08.997 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi123 smithi123 *:9100 running (4m) 28s ago 13m 9.94M - 1.7.0 72c9c2088986 ca7d0b29af81 2024-09-21T10:44:08.997 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi175 smithi175 *:9100 running (3m) 69s ago 12m 11.1M - 1.7.0 72c9c2088986 46c4f6b7c149 2024-09-21T10:44:08.997 INFO:teuthology.orchestra.run.smithi123.stdout:osd.0 smithi123 running (37s) 28s ago 12m 12.2M 1936M 19.3.0-5103-g5c1cdc3c 8f4d013db793 cc40f695c3a8 2024-09-21T10:44:08.997 INFO:teuthology.orchestra.run.smithi123.stdout:osd.1 smithi123 running (11m) 28s ago 11m 519M 1936M 17.2.0 e1d6a67b021e 59aee515b0f8 2024-09-21T10:44:08.997 INFO:teuthology.orchestra.run.smithi123.stdout:osd.2 smithi123 running (11m) 28s ago 11m 415M 1936M 17.2.0 e1d6a67b021e 7db93f8ce5b5 2024-09-21T10:44:08.998 INFO:teuthology.orchestra.run.smithi123.stdout:osd.3 smithi123 running (10m) 28s ago 10m 521M 1936M 17.2.0 e1d6a67b021e c1672dea7b1f 2024-09-21T10:44:08.998 INFO:teuthology.orchestra.run.smithi123.stdout:osd.4 smithi175 running (10m) 69s ago 10m 498M 2704M 17.2.0 e1d6a67b021e 48fe728c259e 2024-09-21T10:44:08.998 INFO:teuthology.orchestra.run.smithi123.stdout:osd.5 smithi175 running (10m) 69s ago 9m 331M 2704M 17.2.0 e1d6a67b021e e270d5a21b14 2024-09-21T10:44:08.998 INFO:teuthology.orchestra.run.smithi123.stdout:osd.6 smithi175 running (9m) 69s ago 9m 517M 2704M 17.2.0 e1d6a67b021e 9bcbf018eb20 2024-09-21T10:44:08.998 INFO:teuthology.orchestra.run.smithi123.stdout:osd.7 smithi175 running (9m) 69s ago 9m 383M 2704M 17.2.0 e1d6a67b021e e9b8f1f90994 2024-09-21T10:44:08.998 INFO:teuthology.orchestra.run.smithi123.stdout:prometheus.smithi123 smithi123 *:9095 running (2m) 28s ago 12m 50.7M - 2.51.0 1d3b7f56885b 37f944c9de88 2024-09-21T10:44:09.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:08 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:44:09.679 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-09-21T10:44:09.680 INFO:teuthology.orchestra.run.smithi123.stdout: "mon": { 2024-09-21T10:44:09.680 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev)": 2 2024-09-21T10:44:09.680 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:44:09.680 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr": { 2024-09-21T10:44:09.680 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev)": 2 2024-09-21T10:44:09.680 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:44:09.680 INFO:teuthology.orchestra.run.smithi123.stdout: "osd": { 2024-09-21T10:44:09.680 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-09-21T10:44:09.680 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev)": 1 2024-09-21T10:44:09.680 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:44:09.680 INFO:teuthology.orchestra.run.smithi123.stdout: "mds": { 2024-09-21T10:44:09.680 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T10:44:09.681 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:44:09.681 INFO:teuthology.orchestra.run.smithi123.stdout: "overall": { 2024-09-21T10:44:09.681 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 9, 2024-09-21T10:44:09.681 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev)": 5 2024-09-21T10:44:09.681 INFO:teuthology.orchestra.run.smithi123.stdout: } 2024-09-21T10:44:09.681 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-09-21T10:44:09.728 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:09 smithi123 ceph-mon[63856]: from='client.34226 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:44:09.728 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:09 smithi123 ceph-mon[63856]: pgmap v90: 97 pgs: 97 active+clean; 347 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.2 MiB/s wr, 507 op/s; 0 B/s, 4 objects/s recovering 2024-09-21T10:44:09.728 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:09 smithi123 ceph-mon[63856]: from='client.34230 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:44:09.728 INFO:journalctl@ceph.osd.1.smithi123.stdout:Sep 21 10:44:09 smithi123 systemd[1]: Stopping Ceph osd.1 for 3ccd15e6-7804-11ef-baf6-efdc52797490... 2024-09-21T10:44:09.831 INFO:teuthology.orchestra.run.smithi123.stdout: 5 179167 10.31 MB/sec execute 310 sec latency 68.886 ms 2024-09-21T10:44:10.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:09 smithi175 ceph-mon[49304]: from='client.34226 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:44:10.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:09 smithi175 ceph-mon[49304]: pgmap v90: 97 pgs: 97 active+clean; 347 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.2 MiB/s wr, 507 op/s; 0 B/s, 4 objects/s recovering 2024-09-21T10:44:10.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:09 smithi175 ceph-mon[49304]: from='client.34230 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:44:10.166 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-09-21T10:44:10.166 INFO:teuthology.orchestra.run.smithi123.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472", 2024-09-21T10:44:10.166 INFO:teuthology.orchestra.run.smithi123.stdout: "in_progress": true, 2024-09-21T10:44:10.166 INFO:teuthology.orchestra.run.smithi123.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T10:44:10.166 INFO:teuthology.orchestra.run.smithi123.stdout: "services_complete": [ 2024-09-21T10:44:10.167 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr", 2024-09-21T10:44:10.167 INFO:teuthology.orchestra.run.smithi123.stdout: "crash", 2024-09-21T10:44:10.167 INFO:teuthology.orchestra.run.smithi123.stdout: "mon" 2024-09-21T10:44:10.167 INFO:teuthology.orchestra.run.smithi123.stdout: ], 2024-09-21T10:44:10.167 INFO:teuthology.orchestra.run.smithi123.stdout: "progress": "7/23 daemons upgraded", 2024-09-21T10:44:10.167 INFO:teuthology.orchestra.run.smithi123.stdout: "message": "Currently upgrading osd daemons", 2024-09-21T10:44:10.167 INFO:teuthology.orchestra.run.smithi123.stdout: "is_paused": false 2024-09-21T10:44:10.167 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-09-21T10:44:10.240 INFO:journalctl@ceph.osd.1.smithi123.stdout:Sep 21 10:44:09 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-1[38220]: 2024-09-21T10:44:09.985+0000 7fb85a66e700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:44:10.240 INFO:journalctl@ceph.osd.1.smithi123.stdout:Sep 21 10:44:09 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-1[38220]: 2024-09-21T10:44:09.985+0000 7fb85a66e700 -1 osd.1 90 *** Got signal Terminated *** 2024-09-21T10:44:10.240 INFO:journalctl@ceph.osd.1.smithi123.stdout:Sep 21 10:44:09 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-1[38220]: 2024-09-21T10:44:09.985+0000 7fb85a66e700 -1 osd.1 90 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T10:44:10.750 INFO:teuthology.orchestra.run.smithi123.stdout:HEALTH_WARN 1 osds down 2024-09-21T10:44:10.751 INFO:teuthology.orchestra.run.smithi123.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-21T10:44:10.751 INFO:teuthology.orchestra.run.smithi123.stdout: osd.1 (root=default,host=smithi123) is down 2024-09-21T10:44:10.776 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:10 smithi123 ceph-mon[63856]: from='client.34234 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:44:10.776 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:10 smithi123 ceph-mon[63856]: from='client.? 172.21.15.123:0/989839504' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:44:10.776 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:10 smithi123 ceph-mon[63856]: osd.1 marked itself down and dead 2024-09-21T10:44:10.831 INFO:teuthology.orchestra.run.smithi123.stdout: 5 179475 10.31 MB/sec execute 311 sec latency 127.575 ms 2024-09-21T10:44:11.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:10 smithi175 ceph-mon[49304]: from='client.34234 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:44:11.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:10 smithi175 ceph-mon[49304]: from='client.? 172.21.15.123:0/989839504' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:44:11.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:10 smithi175 ceph-mon[49304]: osd.1 marked itself down and dead 2024-09-21T10:44:11.600 INFO:journalctl@ceph.osd.1.smithi123.stdout:Sep 21 10:44:11 smithi123 podman[70650]: 2024-09-21 10:44:11.32832594 +0000 UTC m=+1.477069890 container died 59aee515b0f881218795b160d27ae4e76acfbdd0d34c14a206c7265cbd3afc67 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-1, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, version=8, architecture=x86_64, distribution-scope=public, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, RELEASE=HEAD, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, ceph=True, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream) 2024-09-21T10:44:11.831 INFO:teuthology.orchestra.run.smithi123.stdout: 5 179580 10.29 MB/sec execute 312 sec latency 951.528 ms 2024-09-21T10:44:11.850 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:11 smithi123 ceph-mon[63856]: from='client.34242 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:44:11.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:11 smithi123 ceph-mon[63856]: pgmap v91: 97 pgs: 97 active+clean; 341 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 7.0 MiB/s wr, 685 op/s; 0 B/s, 4 objects/s recovering 2024-09-21T10:44:11.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:11 smithi123 ceph-mon[63856]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T10:44:11.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:11 smithi123 ceph-mon[63856]: osdmap e91: 8 total, 7 up, 8 in 2024-09-21T10:44:11.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:11 smithi123 ceph-mon[63856]: from='client.? 172.21.15.123:0/146812869' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T10:44:11.851 INFO:journalctl@ceph.osd.1.smithi123.stdout:Sep 21 10:44:11 smithi123 podman[70650]: 2024-09-21 10:44:11.705656361 +0000 UTC m=+1.854400310 container cleanup 59aee515b0f881218795b160d27ae4e76acfbdd0d34c14a206c7265cbd3afc67 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-1, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, architecture=x86_64, io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, version=8, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git) 2024-09-21T10:44:11.851 INFO:journalctl@ceph.osd.1.smithi123.stdout:Sep 21 10:44:11 smithi123 bash[70650]: ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-1 2024-09-21T10:44:12.101 INFO:journalctl@ceph.osd.1.smithi123.stdout:Sep 21 10:44:12 smithi123 podman[70692]: 2024-09-21 10:44:12.049671465 +0000 UTC m=+0.724066519 container remove 59aee515b0f881218795b160d27ae4e76acfbdd0d34c14a206c7265cbd3afc67 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-1, GIT_BRANCH=HEAD, release=754, io.openshift.expose-services=, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, name=centos-stream, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8) 2024-09-21T10:44:12.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:11 smithi175 ceph-mon[49304]: from='client.34242 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:44:12.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:11 smithi175 ceph-mon[49304]: pgmap v91: 97 pgs: 97 active+clean; 341 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 7.0 MiB/s wr, 685 op/s; 0 B/s, 4 objects/s recovering 2024-09-21T10:44:12.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:11 smithi175 ceph-mon[49304]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T10:44:12.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:11 smithi175 ceph-mon[49304]: osdmap e91: 8 total, 7 up, 8 in 2024-09-21T10:44:12.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:11 smithi175 ceph-mon[49304]: from='client.? 172.21.15.123:0/146812869' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T10:44:12.627 INFO:journalctl@ceph.osd.1.smithi123.stdout:Sep 21 10:44:12 smithi123 podman[70777]: 2024-09-21 10:44:12.499860657 +0000 UTC m=+0.018872387 image pull 8f4d013db793795fef09ff5edf96e6b42ba8cd2c9158b2f0b7a5d7c1e657c063 quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472 2024-09-21T10:44:12.831 INFO:teuthology.orchestra.run.smithi123.stdout: 5 179921 10.29 MB/sec execute 313 sec latency 64.657 ms 2024-09-21T10:44:12.879 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:12 smithi123 ceph-mon[63856]: osdmap e92: 8 total, 7 up, 8 in 2024-09-21T10:44:12.879 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:12 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:44:12.879 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:12 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:44:12.879 INFO:journalctl@ceph.osd.1.smithi123.stdout:Sep 21 10:44:12 smithi123 podman[70777]: 2024-09-21 10:44:12.634050841 +0000 UTC m=+0.153062555 container create ab0b9fd9a3d12cccfa9a9f3d75bf00080d230c057b4eb0b4ede6263849514036 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-1-deactivate, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True) 2024-09-21T10:44:13.024 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:12 smithi175 ceph-mon[49304]: osdmap e92: 8 total, 7 up, 8 in 2024-09-21T10:44:13.024 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:12 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:44:13.024 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:12 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:44:13.136 INFO:journalctl@ceph.osd.1.smithi123.stdout:Sep 21 10:44:13 smithi123 podman[70777]: 2024-09-21 10:44:13.063099431 +0000 UTC m=+0.582111146 container init ab0b9fd9a3d12cccfa9a9f3d75bf00080d230c057b4eb0b4ede6263849514036 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-1-deactivate, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3) 2024-09-21T10:44:13.136 INFO:journalctl@ceph.osd.1.smithi123.stdout:Sep 21 10:44:13 smithi123 podman[70777]: 2024-09-21 10:44:13.073852292 +0000 UTC m=+0.592864015 container start ab0b9fd9a3d12cccfa9a9f3d75bf00080d230c057b4eb0b4ede6263849514036 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-1-deactivate, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD) 2024-09-21T10:44:13.387 INFO:journalctl@ceph.osd.1.smithi123.stdout:Sep 21 10:44:13 smithi123 podman[70777]: 2024-09-21 10:44:13.171464133 +0000 UTC m=+0.690475863 container attach ab0b9fd9a3d12cccfa9a9f3d75bf00080d230c057b4eb0b4ede6263849514036 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-1-deactivate, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-09-21T10:44:13.387 INFO:journalctl@ceph.osd.1.smithi123.stdout:Sep 21 10:44:13 smithi123 podman[70777]: 2024-09-21 10:44:13.383667799 +0000 UTC m=+0.902679517 container died ab0b9fd9a3d12cccfa9a9f3d75bf00080d230c057b4eb0b4ede6263849514036 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-1-deactivate, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2) 2024-09-21T10:44:13.831 INFO:teuthology.orchestra.run.smithi123.stdout: 5 180306 10.29 MB/sec execute 314 sec latency 60.064 ms 2024-09-21T10:44:13.891 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:13 smithi123 ceph-mon[63856]: pgmap v94: 97 pgs: 14 stale+active+clean, 83 active+clean; 341 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 3.0 MiB/s wr, 287 op/s 2024-09-21T10:44:14.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:13 smithi175 ceph-mon[49304]: pgmap v94: 97 pgs: 14 stale+active+clean, 83 active+clean; 341 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 3.0 MiB/s wr, 287 op/s 2024-09-21T10:44:14.414 INFO:journalctl@ceph.osd.1.smithi123.stdout:Sep 21 10:44:14 smithi123 podman[70814]: 2024-09-21 10:44:14.367502013 +0000 UTC m=+0.976432277 container remove ab0b9fd9a3d12cccfa9a9f3d75bf00080d230c057b4eb0b4ede6263849514036 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-1-deactivate, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, ceph=True) 2024-09-21T10:44:14.414 INFO:journalctl@ceph.osd.1.smithi123.stdout:Sep 21 10:44:14 smithi123 systemd[1]: ceph-3ccd15e6-7804-11ef-baf6-efdc52797490@osd.1.service: Deactivated successfully. 2024-09-21T10:44:14.414 INFO:journalctl@ceph.osd.1.smithi123.stdout:Sep 21 10:44:14 smithi123 systemd[1]: Stopped Ceph osd.1 for 3ccd15e6-7804-11ef-baf6-efdc52797490. 2024-09-21T10:44:14.414 INFO:journalctl@ceph.osd.1.smithi123.stdout:Sep 21 10:44:14 smithi123 systemd[1]: ceph-3ccd15e6-7804-11ef-baf6-efdc52797490@osd.1.service: Consumed 3min 5.987s CPU time. 2024-09-21T10:44:14.831 INFO:teuthology.orchestra.run.smithi123.stdout: 5 180582 10.27 MB/sec execute 315 sec latency 66.771 ms 2024-09-21T10:44:14.922 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:14 smithi123 ceph-mon[63856]: Health check failed: Degraded data redundancy: 160/1590 objects degraded (10.063%), 21 pgs degraded (PG_DEGRADED) 2024-09-21T10:44:14.922 INFO:journalctl@ceph.osd.1.smithi123.stdout:Sep 21 10:44:14 smithi123 systemd[1]: Starting Ceph osd.1 for 3ccd15e6-7804-11ef-baf6-efdc52797490... 2024-09-21T10:44:14.922 INFO:journalctl@ceph.osd.1.smithi123.stdout:Sep 21 10:44:14 smithi123 podman[70911]: 2024-09-21 10:44:14.795496766 +0000 UTC m=+0.019516471 image pull 8f4d013db793795fef09ff5edf96e6b42ba8cd2c9158b2f0b7a5d7c1e657c063 quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472 2024-09-21T10:44:15.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:14 smithi175 ceph-mon[49304]: Health check failed: Degraded data redundancy: 160/1590 objects degraded (10.063%), 21 pgs degraded (PG_DEGRADED) 2024-09-21T10:44:15.174 INFO:journalctl@ceph.osd.1.smithi123.stdout:Sep 21 10:44:14 smithi123 podman[70911]: 2024-09-21 10:44:14.948990797 +0000 UTC m=+0.173010482 container create 513e967310d0ecb4f1e67d92517fe88ba16a5812e8991cd51e1b4e4a3006a3bd (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-1-activate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2) 2024-09-21T10:44:15.427 INFO:journalctl@ceph.osd.1.smithi123.stdout:Sep 21 10:44:15 smithi123 podman[70911]: 2024-09-21 10:44:15.350694556 +0000 UTC m=+0.574714254 container init 513e967310d0ecb4f1e67d92517fe88ba16a5812e8991cd51e1b4e4a3006a3bd (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-1-activate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_BRANCH=HEAD, ceph=True) 2024-09-21T10:44:15.427 INFO:journalctl@ceph.osd.1.smithi123.stdout:Sep 21 10:44:15 smithi123 podman[70911]: 2024-09-21 10:44:15.355835124 +0000 UTC m=+0.579854809 container start 513e967310d0ecb4f1e67d92517fe88ba16a5812e8991cd51e1b4e4a3006a3bd (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-1-activate, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-09-21T10:44:15.690 INFO:journalctl@ceph.osd.1.smithi123.stdout:Sep 21 10:44:15 smithi123 podman[70911]: 2024-09-21 10:44:15.459114611 +0000 UTC m=+0.683134303 container attach 513e967310d0ecb4f1e67d92517fe88ba16a5812e8991cd51e1b4e4a3006a3bd (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-1-activate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240731) 2024-09-21T10:44:15.690 INFO:journalctl@ceph.osd.1.smithi123.stdout:Sep 21 10:44:15 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-1-activate[70925]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:44:15.690 INFO:journalctl@ceph.osd.1.smithi123.stdout:Sep 21 10:44:15 smithi123 bash[70911]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:44:15.691 INFO:journalctl@ceph.osd.1.smithi123.stdout:Sep 21 10:44:15 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-1-activate[70925]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:44:15.691 INFO:journalctl@ceph.osd.1.smithi123.stdout:Sep 21 10:44:15 smithi123 bash[70911]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:44:15.831 INFO:teuthology.orchestra.run.smithi123.stdout: 5 180780 10.26 MB/sec execute 316 sec latency 65.590 ms 2024-09-21T10:44:15.941 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:15 smithi123 ceph-mon[63856]: pgmap v95: 97 pgs: 14 active+undersized, 1 stale+active+clean, 21 active+undersized+degraded, 61 active+clean; 341 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 4.6 MiB/s wr, 496 op/s; 160/1590 objects degraded (10.063%) 2024-09-21T10:44:16.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:15 smithi175 ceph-mon[49304]: pgmap v95: 97 pgs: 14 active+undersized, 1 stale+active+clean, 21 active+undersized+degraded, 61 active+clean; 341 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 4.6 MiB/s wr, 496 op/s; 160/1590 objects degraded (10.063%) 2024-09-21T10:44:16.831 INFO:teuthology.orchestra.run.smithi123.stdout: 5 181098 10.27 MB/sec execute 317 sec latency 120.041 ms 2024-09-21T10:44:17.734 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:17 smithi123 ceph-mon[63856]: pgmap v96: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 342 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.1 MiB/s wr, 518 op/s; 178/1605 objects degraded (11.090%) 2024-09-21T10:44:17.831 INFO:teuthology.orchestra.run.smithi123.stdout: 5 181455 10.26 MB/sec execute 318 sec latency 44.840 ms 2024-09-21T10:44:18.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:17 smithi175 ceph-mon[49304]: pgmap v96: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 342 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.1 MiB/s wr, 518 op/s; 178/1605 objects degraded (11.090%) 2024-09-21T10:44:18.831 INFO:teuthology.orchestra.run.smithi123.stdout: 5 182066 10.28 MB/sec execute 319 sec latency 36.358 ms 2024-09-21T10:44:19.758 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:19 smithi123 ceph-mon[63856]: pgmap v97: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 342 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 3.7 KiB/s rd, 2.5 MiB/s wr, 251 op/s; 178/1605 objects degraded (11.090%) 2024-09-21T10:44:19.831 INFO:teuthology.orchestra.run.smithi123.stdout: 5 182673 10.29 MB/sec execute 320 sec latency 30.820 ms 2024-09-21T10:44:20.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:19 smithi175 ceph-mon[49304]: pgmap v97: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 342 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 3.7 KiB/s rd, 2.5 MiB/s wr, 251 op/s; 178/1605 objects degraded (11.090%) 2024-09-21T10:44:20.275 INFO:journalctl@ceph.osd.1.smithi123.stdout:Sep 21 10:44:20 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-1-activate[70925]: --> Failed to activate via raw: did not find any matching OSD to activate 2024-09-21T10:44:20.275 INFO:journalctl@ceph.osd.1.smithi123.stdout:Sep 21 10:44:20 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-1-activate[70925]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:44:20.275 INFO:journalctl@ceph.osd.1.smithi123.stdout:Sep 21 10:44:20 smithi123 bash[70911]: --> Failed to activate via raw: did not find any matching OSD to activate 2024-09-21T10:44:20.275 INFO:journalctl@ceph.osd.1.smithi123.stdout:Sep 21 10:44:20 smithi123 bash[70911]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:44:20.276 INFO:journalctl@ceph.osd.1.smithi123.stdout:Sep 21 10:44:20 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-1-activate[70925]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:44:20.276 INFO:journalctl@ceph.osd.1.smithi123.stdout:Sep 21 10:44:20 smithi123 bash[70911]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:44:20.526 INFO:journalctl@ceph.osd.1.smithi123.stdout:Sep 21 10:44:20 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-1-activate[70925]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-21T10:44:20.526 INFO:journalctl@ceph.osd.1.smithi123.stdout:Sep 21 10:44:20 smithi123 bash[70911]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-21T10:44:20.526 INFO:journalctl@ceph.osd.1.smithi123.stdout:Sep 21 10:44:20 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-1-activate[70925]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-1 --no-mon-config 2024-09-21T10:44:20.526 INFO:journalctl@ceph.osd.1.smithi123.stdout:Sep 21 10:44:20 smithi123 bash[70911]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-1 --no-mon-config 2024-09-21T10:44:20.778 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:20 smithi123 ceph-mon[63856]: Health check update: Degraded data redundancy: 178/1605 objects degraded (11.090%), 23 pgs degraded (PG_DEGRADED) 2024-09-21T10:44:20.779 INFO:journalctl@ceph.osd.1.smithi123.stdout:Sep 21 10:44:20 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-1-activate[70925]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-1/block 2024-09-21T10:44:20.779 INFO:journalctl@ceph.osd.1.smithi123.stdout:Sep 21 10:44:20 smithi123 bash[70911]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-1/block 2024-09-21T10:44:20.779 INFO:journalctl@ceph.osd.1.smithi123.stdout:Sep 21 10:44:20 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-1-activate[70925]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-1/block 2024-09-21T10:44:20.779 INFO:journalctl@ceph.osd.1.smithi123.stdout:Sep 21 10:44:20 smithi123 bash[70911]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-1/block 2024-09-21T10:44:20.779 INFO:journalctl@ceph.osd.1.smithi123.stdout:Sep 21 10:44:20 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-1-activate[70925]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-21T10:44:20.779 INFO:journalctl@ceph.osd.1.smithi123.stdout:Sep 21 10:44:20 smithi123 bash[70911]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-21T10:44:20.779 INFO:journalctl@ceph.osd.1.smithi123.stdout:Sep 21 10:44:20 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-1-activate[70925]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-21T10:44:20.779 INFO:journalctl@ceph.osd.1.smithi123.stdout:Sep 21 10:44:20 smithi123 bash[70911]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-21T10:44:20.779 INFO:journalctl@ceph.osd.1.smithi123.stdout:Sep 21 10:44:20 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-1-activate[70925]: --> ceph-volume lvm activate successful for osd ID: 1 2024-09-21T10:44:20.780 INFO:journalctl@ceph.osd.1.smithi123.stdout:Sep 21 10:44:20 smithi123 bash[70911]: --> ceph-volume lvm activate successful for osd ID: 1 2024-09-21T10:44:20.831 INFO:teuthology.orchestra.run.smithi123.stdout: 5 182991 10.28 MB/sec execute 321 sec latency 44.353 ms 2024-09-21T10:44:21.034 INFO:journalctl@ceph.osd.1.smithi123.stdout:Sep 21 10:44:20 smithi123 podman[70911]: 2024-09-21 10:44:20.779247477 +0000 UTC m=+6.003267177 container died 513e967310d0ecb4f1e67d92517fe88ba16a5812e8991cd51e1b4e4a3006a3bd (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-1-activate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-09-21T10:44:21.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:20 smithi175 ceph-mon[49304]: Health check update: Degraded data redundancy: 178/1605 objects degraded (11.090%), 23 pgs degraded (PG_DEGRADED) 2024-09-21T10:44:21.795 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:21 smithi123 ceph-mon[63856]: pgmap v98: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 347 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.7 MiB/s wr, 448 op/s; 182/1701 objects degraded (10.700%) 2024-09-21T10:44:21.795 INFO:journalctl@ceph.osd.1.smithi123.stdout:Sep 21 10:44:21 smithi123 podman[71147]: 2024-09-21 10:44:21.651105157 +0000 UTC m=+0.860187322 container remove 513e967310d0ecb4f1e67d92517fe88ba16a5812e8991cd51e1b4e4a3006a3bd (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-1-activate, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731) 2024-09-21T10:44:21.831 INFO:teuthology.orchestra.run.smithi123.stdout: 5 183322 10.28 MB/sec execute 322 sec latency 44.107 ms 2024-09-21T10:44:22.060 INFO:journalctl@ceph.osd.1.smithi123.stdout:Sep 21 10:44:21 smithi123 podman[71200]: 2024-09-21 10:44:21.820300054 +0000 UTC m=+0.029226809 image pull 8f4d013db793795fef09ff5edf96e6b42ba8cd2c9158b2f0b7a5d7c1e657c063 quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472 2024-09-21T10:44:22.061 INFO:journalctl@ceph.osd.1.smithi123.stdout:Sep 21 10:44:21 smithi123 podman[71200]: 2024-09-21 10:44:21.958227821 +0000 UTC m=+0.167154575 container create 1be249025c7f0b374c5328a4e25918f320c228f97c09cb5c6026bdb56cdedb07 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-1, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_CLEAN=True) 2024-09-21T10:44:22.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:21 smithi175 ceph-mon[49304]: pgmap v98: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 347 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.7 MiB/s wr, 448 op/s; 182/1701 objects degraded (10.700%) 2024-09-21T10:44:22.312 INFO:journalctl@ceph.osd.1.smithi123.stdout:Sep 21 10:44:22 smithi123 podman[71200]: 2024-09-21 10:44:22.269874422 +0000 UTC m=+0.478801196 container init 1be249025c7f0b374c5328a4e25918f320c228f97c09cb5c6026bdb56cdedb07 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-1, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, org.label-schema.schema-version=1.0) 2024-09-21T10:44:22.312 INFO:journalctl@ceph.osd.1.smithi123.stdout:Sep 21 10:44:22 smithi123 podman[71200]: 2024-09-21 10:44:22.28734011 +0000 UTC m=+0.496266865 container start 1be249025c7f0b374c5328a4e25918f320c228f97c09cb5c6026bdb56cdedb07 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-1, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-09-21T10:44:22.569 INFO:journalctl@ceph.osd.1.smithi123.stdout:Sep 21 10:44:22 smithi123 ceph-osd[71218]: -- 172.21.15.123:0/800125597 <== mon.1 v2:172.21.15.175:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x56297a8841e0 con 0x56297a86c000 2024-09-21T10:44:22.569 INFO:journalctl@ceph.osd.1.smithi123.stdout:Sep 21 10:44:22 smithi123 bash[71200]: 1be249025c7f0b374c5328a4e25918f320c228f97c09cb5c6026bdb56cdedb07 2024-09-21T10:44:22.569 INFO:journalctl@ceph.osd.1.smithi123.stdout:Sep 21 10:44:22 smithi123 systemd[1]: Started Ceph osd.1 for 3ccd15e6-7804-11ef-baf6-efdc52797490. 2024-09-21T10:44:22.831 INFO:teuthology.orchestra.run.smithi123.stdout: 5 183656 10.27 MB/sec execute 323 sec latency 53.887 ms 2024-09-21T10:44:23.578 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:23 smithi123 ceph-mon[63856]: pgmap v99: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 347 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.3 MiB/s wr, 406 op/s; 182/1701 objects degraded (10.700%) 2024-09-21T10:44:23.578 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:23 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:44:23.579 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:23 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:44:23.579 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:23 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:44:23.831 INFO:teuthology.orchestra.run.smithi123.stdout: 5 184327 10.30 MB/sec execute 324 sec latency 35.513 ms 2024-09-21T10:44:23.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:23 smithi175 ceph-mon[49304]: pgmap v99: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 347 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.3 MiB/s wr, 406 op/s; 182/1701 objects degraded (10.700%) 2024-09-21T10:44:23.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:23 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:44:23.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:23 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:44:23.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:23 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:44:24.332 INFO:journalctl@ceph.osd.1.smithi123.stdout:Sep 21 10:44:24 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-1[71214]: 2024-09-21T10:44:24.230+0000 7f41a168f640 -1 Falling back to public interface 2024-09-21T10:44:24.831 INFO:teuthology.orchestra.run.smithi123.stdout: 5 185019 10.33 MB/sec execute 325 sec latency 32.891 ms 2024-09-21T10:44:25.598 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:25 smithi123 ceph-mon[63856]: pgmap v100: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 346 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 6.1 MiB/s wr, 617 op/s; 177/1668 objects degraded (10.612%) 2024-09-21T10:44:25.598 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:25 smithi123 ceph-mon[63856]: Health check update: Degraded data redundancy: 177/1668 objects degraded (10.612%), 23 pgs degraded (PG_DEGRADED) 2024-09-21T10:44:25.831 INFO:teuthology.orchestra.run.smithi123.stdout: 5 185566 10.35 MB/sec execute 326 sec latency 34.241 ms 2024-09-21T10:44:25.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:25 smithi175 ceph-mon[49304]: pgmap v100: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 346 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 6.1 MiB/s wr, 617 op/s; 177/1668 objects degraded (10.612%) 2024-09-21T10:44:25.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:25 smithi175 ceph-mon[49304]: Health check update: Degraded data redundancy: 177/1668 objects degraded (10.612%), 23 pgs degraded (PG_DEGRADED) 2024-09-21T10:44:26.831 INFO:teuthology.orchestra.run.smithi123.stdout: 5 186155 10.36 MB/sec execute 327 sec latency 31.344 ms 2024-09-21T10:44:27.698 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:27 smithi123 ceph-mon[63856]: pgmap v101: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 345 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 5.1 MiB/s wr, 481 op/s; 176/1674 objects degraded (10.514%) 2024-09-21T10:44:27.831 INFO:teuthology.orchestra.run.smithi123.stdout: 5 186535 10.35 MB/sec execute 328 sec latency 102.379 ms 2024-09-21T10:44:27.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:27 smithi175 ceph-mon[49304]: pgmap v101: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 345 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 5.1 MiB/s wr, 481 op/s; 176/1674 objects degraded (10.514%) 2024-09-21T10:44:28.740 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:28 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:44:28.740 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:28 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:44:28.831 INFO:teuthology.orchestra.run.smithi123.stdout: 5 186870 10.36 MB/sec execute 329 sec latency 39.570 ms 2024-09-21T10:44:29.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:28 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:44:29.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:28 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:44:29.760 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:29 smithi123 ceph-mon[63856]: pgmap v102: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 345 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.7 MiB/s wr, 466 op/s; 176/1674 objects degraded (10.514%) 2024-09-21T10:44:29.831 INFO:teuthology.orchestra.run.smithi123.stdout: 5 187221 10.35 MB/sec execute 330 sec latency 56.033 ms 2024-09-21T10:44:30.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:29 smithi175 ceph-mon[49304]: pgmap v102: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 345 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.7 MiB/s wr, 466 op/s; 176/1674 objects degraded (10.514%) 2024-09-21T10:44:30.812 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:30 smithi123 ceph-mon[63856]: Health check update: Degraded data redundancy: 176/1674 objects degraded (10.514%), 23 pgs degraded (PG_DEGRADED) 2024-09-21T10:44:30.831 INFO:teuthology.orchestra.run.smithi123.stdout: 5 187546 10.34 MB/sec execute 331 sec latency 100.807 ms 2024-09-21T10:44:31.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:30 smithi175 ceph-mon[49304]: Health check update: Degraded data redundancy: 176/1674 objects degraded (10.514%), 23 pgs degraded (PG_DEGRADED) 2024-09-21T10:44:31.829 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:31 smithi123 ceph-mon[63856]: pgmap v103: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 347 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 8.2 MiB/s wr, 806 op/s; 165/1635 objects degraded (10.092%) 2024-09-21T10:44:31.829 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:31 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:44:31.829 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:31 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:44:31.831 INFO:teuthology.orchestra.run.smithi123.stdout: 5 187768 10.33 MB/sec execute 332 sec latency 66.318 ms 2024-09-21T10:44:32.084 INFO:journalctl@ceph.osd.1.smithi123.stdout:Sep 21 10:44:31 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-1[71214]: 2024-09-21T10:44:31.879+0000 7f41a168f640 -1 osd.1 0 read_superblock omap replica is missing. 2024-09-21T10:44:32.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:31 smithi175 ceph-mon[49304]: pgmap v103: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 347 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 8.2 MiB/s wr, 806 op/s; 165/1635 objects degraded (10.092%) 2024-09-21T10:44:32.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:31 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:44:32.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:31 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:44:32.832 INFO:teuthology.orchestra.run.smithi123.stdout: 5 188038 10.33 MB/sec execute 333 sec latency 42.412 ms 2024-09-21T10:44:32.851 INFO:journalctl@ceph.osd.1.smithi123.stdout:Sep 21 10:44:32 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-1[71214]: 2024-09-21T10:44:32.773+0000 7f41a168f640 -1 osd.1 90 log_to_monitors true 2024-09-21T10:44:33.114 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:32 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:44:33.115 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:32 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:44:33.115 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:32 smithi123 ceph-mon[63856]: pgmap v104: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 347 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 6.1 MiB/s wr, 615 op/s; 165/1635 objects degraded (10.092%) 2024-09-21T10:44:33.115 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:32 smithi123 ceph-mon[63856]: from='osd.1 [v2:172.21.15.123:6810/4164408454,v1:172.21.15.123:6811/4164408454]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-21T10:44:33.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:32 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:44:33.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:32 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:44:33.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:32 smithi175 ceph-mon[49304]: pgmap v104: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 347 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 6.1 MiB/s wr, 615 op/s; 165/1635 objects degraded (10.092%) 2024-09-21T10:44:33.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:32 smithi175 ceph-mon[49304]: from='osd.1 [v2:172.21.15.123:6810/4164408454,v1:172.21.15.123:6811/4164408454]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-21T10:44:33.618 INFO:journalctl@ceph.osd.1.smithi123.stdout:Sep 21 10:44:33 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-1[71214]: 2024-09-21T10:44:33.510+0000 7f4195431640 -1 osd.1 90 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T10:44:33.832 INFO:teuthology.orchestra.run.smithi123.stdout: 5 188295 10.33 MB/sec execute 334 sec latency 88.582 ms 2024-09-21T10:44:34.132 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:33 smithi123 ceph-mon[63856]: from='osd.1 [v2:172.21.15.123:6810/4164408454,v1:172.21.15.123:6811/4164408454]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-21T10:44:34.132 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:33 smithi123 ceph-mon[63856]: osdmap e93: 8 total, 7 up, 8 in 2024-09-21T10:44:34.133 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:33 smithi123 ceph-mon[63856]: from='osd.1 [v2:172.21.15.123:6810/4164408454,v1:172.21.15.123:6811/4164408454]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi123", "root=default"]} : dispatch 2024-09-21T10:44:34.133 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:33 smithi123 ceph-mon[63856]: from='osd.1 [v2:172.21.15.123:6810/4164408454,v1:172.21.15.123:6811/4164408454]' entity='osd.1' 2024-09-21T10:44:34.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:33 smithi175 ceph-mon[49304]: from='osd.1 [v2:172.21.15.123:6810/4164408454,v1:172.21.15.123:6811/4164408454]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-21T10:44:34.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:33 smithi175 ceph-mon[49304]: osdmap e93: 8 total, 7 up, 8 in 2024-09-21T10:44:34.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:33 smithi175 ceph-mon[49304]: from='osd.1 [v2:172.21.15.123:6810/4164408454,v1:172.21.15.123:6811/4164408454]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi123", "root=default"]} : dispatch 2024-09-21T10:44:34.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:33 smithi175 ceph-mon[49304]: from='osd.1 [v2:172.21.15.123:6810/4164408454,v1:172.21.15.123:6811/4164408454]' entity='osd.1' 2024-09-21T10:44:34.832 INFO:teuthology.orchestra.run.smithi123.stdout: 5 188595 10.31 MB/sec execute 335 sec latency 338.431 ms 2024-09-21T10:44:35.145 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:35 smithi123 ceph-mon[63856]: pgmap v106: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 6.4 MiB/s wr, 626 op/s; 162/1590 objects degraded (10.189%) 2024-09-21T10:44:35.145 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:35 smithi123 ceph-mon[63856]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T10:44:35.145 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:35 smithi123 ceph-mon[63856]: osd.1 [v2:172.21.15.123:6810/4164408454,v1:172.21.15.123:6811/4164408454] boot 2024-09-21T10:44:35.145 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:35 smithi123 ceph-mon[63856]: osdmap e94: 8 total, 8 up, 8 in 2024-09-21T10:44:35.146 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:35 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-21T10:44:35.146 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:35 smithi123 ceph-mon[63856]: Health check update: Degraded data redundancy: 165/1635 objects degraded (10.092%), 23 pgs degraded (PG_DEGRADED) 2024-09-21T10:44:35.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:35 smithi175 ceph-mon[49304]: pgmap v106: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 6.4 MiB/s wr, 626 op/s; 162/1590 objects degraded (10.189%) 2024-09-21T10:44:35.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:35 smithi175 ceph-mon[49304]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T10:44:35.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:35 smithi175 ceph-mon[49304]: osd.1 [v2:172.21.15.123:6810/4164408454,v1:172.21.15.123:6811/4164408454] boot 2024-09-21T10:44:35.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:35 smithi175 ceph-mon[49304]: osdmap e94: 8 total, 8 up, 8 in 2024-09-21T10:44:35.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:35 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-21T10:44:35.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:35 smithi175 ceph-mon[49304]: Health check update: Degraded data redundancy: 165/1635 objects degraded (10.092%), 23 pgs degraded (PG_DEGRADED) 2024-09-21T10:44:35.832 INFO:teuthology.orchestra.run.smithi123.stdout: 5 188686 10.28 MB/sec execute 336 sec latency 1338.460 ms 2024-09-21T10:44:36.263 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:36 smithi123 ceph-mon[63856]: osdmap e95: 8 total, 8 up, 8 in 2024-09-21T10:44:36.263 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:36 smithi123 ceph-mon[63856]: osdmap e96: 8 total, 8 up, 8 in 2024-09-21T10:44:36.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:36 smithi175 ceph-mon[49304]: osdmap e95: 8 total, 8 up, 8 in 2024-09-21T10:44:36.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:36 smithi175 ceph-mon[49304]: osdmap e96: 8 total, 8 up, 8 in 2024-09-21T10:44:36.832 INFO:teuthology.orchestra.run.smithi123.stdout: 5 188899 10.27 MB/sec execute 337 sec latency 1697.041 ms 2024-09-21T10:44:37.028 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:37 smithi123 ceph-mon[63856]: pgmap v110: 97 pgs: 3 peering, 14 active+undersized, 22 active+undersized+degraded, 58 active+clean; 345 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.3 MiB/s wr, 523 op/s; 151/1593 objects degraded (9.479%) 2024-09-21T10:44:37.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:37 smithi175 ceph-mon[49304]: pgmap v110: 97 pgs: 3 peering, 14 active+undersized, 22 active+undersized+degraded, 58 active+clean; 345 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.3 MiB/s wr, 523 op/s; 151/1593 objects degraded (9.479%) 2024-09-21T10:44:37.832 INFO:teuthology.orchestra.run.smithi123.stdout: 5 189077 10.26 MB/sec execute 338 sec latency 805.022 ms 2024-09-21T10:44:38.107 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:38 smithi123 ceph-mon[63856]: osdmap e97: 8 total, 8 up, 8 in 2024-09-21T10:44:38.107 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:38 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:44:38.107 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:38 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:44:38.107 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:38 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:44:38.107 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:38 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:44:38.108 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:38 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:44:38.108 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:38 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:44:38.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:38 smithi175 ceph-mon[49304]: osdmap e97: 8 total, 8 up, 8 in 2024-09-21T10:44:38.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:38 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:44:38.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:38 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:44:38.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:38 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:44:38.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:38 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:44:38.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:38 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:44:38.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:38 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:44:38.832 INFO:teuthology.orchestra.run.smithi123.stdout: 5 189403 10.26 MB/sec execute 339 sec latency 1021.401 ms 2024-09-21T10:44:39.123 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:39 smithi123 ceph-mon[63856]: osdmap e98: 8 total, 8 up, 8 in 2024-09-21T10:44:39.123 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:39 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:44:39.123 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:39 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:44:39.124 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:39 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:44:39.124 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:39 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:44:39.124 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:39 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-21T10:44:39.124 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:39 smithi123 ceph-mon[63856]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-21T10:44:39.124 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:39 smithi123 ceph-mon[63856]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-09-21T10:44:39.124 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:39 smithi123 ceph-mon[63856]: pgmap v113: 97 pgs: 1 activating, 3 peering, 9 active+undersized, 15 active+undersized+degraded, 69 active+clean; 345 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 788 KiB/s wr, 32 op/s; 118/1593 objects degraded (7.407%); 1.1 MiB/s, 82 keys/s, 7 objects/s recovering 2024-09-21T10:44:39.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:39 smithi175 ceph-mon[49304]: osdmap e98: 8 total, 8 up, 8 in 2024-09-21T10:44:39.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:39 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:44:39.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:39 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:44:39.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:39 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:44:39.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:39 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:44:39.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:39 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-21T10:44:39.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:39 smithi175 ceph-mon[49304]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-21T10:44:39.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:39 smithi175 ceph-mon[49304]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-09-21T10:44:39.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:39 smithi175 ceph-mon[49304]: pgmap v113: 97 pgs: 1 activating, 3 peering, 9 active+undersized, 15 active+undersized+degraded, 69 active+clean; 345 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 788 KiB/s wr, 32 op/s; 118/1593 objects degraded (7.407%); 1.1 MiB/s, 82 keys/s, 7 objects/s recovering 2024-09-21T10:44:39.832 INFO:teuthology.orchestra.run.smithi123.stdout: 5 189848 10.25 MB/sec execute 340 sec latency 551.909 ms 2024-09-21T10:44:40.131 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:40 smithi123 ceph-mon[63856]: osdmap e99: 8 total, 8 up, 8 in 2024-09-21T10:44:40.131 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:40 smithi123 ceph-mon[63856]: Health check update: Degraded data redundancy: 118/1593 objects degraded (7.407%), 15 pgs degraded (PG_DEGRADED) 2024-09-21T10:44:40.131 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:40 smithi123 ceph-mon[63856]: osdmap e100: 8 total, 8 up, 8 in 2024-09-21T10:44:40.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:40 smithi175 ceph-mon[49304]: osdmap e99: 8 total, 8 up, 8 in 2024-09-21T10:44:40.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:40 smithi175 ceph-mon[49304]: Health check update: Degraded data redundancy: 118/1593 objects degraded (7.407%), 15 pgs degraded (PG_DEGRADED) 2024-09-21T10:44:40.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:40 smithi175 ceph-mon[49304]: osdmap e100: 8 total, 8 up, 8 in 2024-09-21T10:44:40.832 INFO:teuthology.orchestra.run.smithi123.stdout: 5 190230 10.25 MB/sec execute 341 sec latency 740.377 ms 2024-09-21T10:44:41.063 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:41 smithi123 ceph-mon[63856]: pgmap v116: 97 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 1 activating, 1 peering, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 342 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 3.9 MiB/s wr, 400 op/s; 169/1581 objects degraded (10.689%); 2.0 MiB/s, 77 keys/s, 28 objects/s recovering 2024-09-21T10:44:41.063 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:41 smithi123 ceph-mon[63856]: osdmap e101: 8 total, 8 up, 8 in 2024-09-21T10:44:41.372 INFO:teuthology.orchestra.run.smithi123.stdout:true 2024-09-21T10:44:41.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:41 smithi175 ceph-mon[49304]: pgmap v116: 97 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 1 activating, 1 peering, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 342 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 3.9 MiB/s wr, 400 op/s; 169/1581 objects degraded (10.689%); 2.0 MiB/s, 77 keys/s, 28 objects/s recovering 2024-09-21T10:44:41.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:41 smithi175 ceph-mon[49304]: osdmap e101: 8 total, 8 up, 8 in 2024-09-21T10:44:41.832 INFO:teuthology.orchestra.run.smithi123.stdout: 5 190544 10.27 MB/sec execute 342 sec latency 824.966 ms 2024-09-21T10:44:42.100 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:42 smithi123 ceph-mon[63856]: from='client.34254 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:44:42.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:42 smithi123 ceph-mon[63856]: osdmap e102: 8 total, 8 up, 8 in 2024-09-21T10:44:42.285 INFO:teuthology.orchestra.run.smithi123.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T10:44:42.286 INFO:teuthology.orchestra.run.smithi123.stdout:alertmanager.smithi123 smithi123 *:9093,9094 running (5m) 11s ago 14m 22.0M - 0.27.0 11f11916f8cd 5d550bd97de5 2024-09-21T10:44:42.286 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi123 smithi123 running (113s) 11s ago 14m 6899k - 19.3.0-5103-g5c1cdc3c 8f4d013db793 c5e1ef39a0a8 2024-09-21T10:44:42.286 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi175 smithi175 running (107s) 102s ago 13m 6895k - 19.3.0-5103-g5c1cdc3c 8f4d013db793 fb1ca999ccfd 2024-09-21T10:44:42.286 INFO:teuthology.orchestra.run.smithi123.stdout:grafana.smithi123 smithi123 *:3000 running (5m) 11s ago 14m 45.3M - dad864ee21e9 771df81763ba 2024-09-21T10:44:42.286 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi123.uqoyow smithi123 running (8m) 11s ago 8m 41.3M - 17.2.0 e1d6a67b021e f0a21eaac266 2024-09-21T10:44:42.286 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi175.jrlevw smithi175 running (8m) 102s ago 8m 13.5M - 17.2.0 e1d6a67b021e b21b16f4bc50 2024-09-21T10:44:42.286 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi123.stnuit smithi123 *:8443,9283,8765 running (4m) 11s ago 15m 517M - 19.3.0-5103-g5c1cdc3c 8f4d013db793 6812cc5cb56b 2024-09-21T10:44:42.286 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi175.txvwrq smithi175 *:8443,9283,8765 running (3m) 102s ago 13m 453M - 19.3.0-5103-g5c1cdc3c 8f4d013db793 304f0408075e 2024-09-21T10:44:42.286 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi123 smithi123 running (3m) 11s ago 15m 48.7M 2048M 19.3.0-5103-g5c1cdc3c 8f4d013db793 6b95207a2d80 2024-09-21T10:44:42.286 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi175 smithi175 running (2m) 102s ago 13m 36.7M 2048M 19.3.0-5103-g5c1cdc3c 8f4d013db793 3abee2d37f0f 2024-09-21T10:44:42.286 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.0.0.smithi123.ddatpr smithi123 *:2049 running (7m) 11s ago 7m 279M - 4.0 e1d6a67b021e 36f8cb2281eb 2024-09-21T10:44:42.286 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.1.0.smithi175.wpawer smithi175 *:2049 running (7m) 102s ago 7m 90.3M - 4.0 e1d6a67b021e 2200aeed9200 2024-09-21T10:44:42.286 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi123 smithi123 *:9100 running (4m) 11s ago 14m 10.3M - 1.7.0 72c9c2088986 ca7d0b29af81 2024-09-21T10:44:42.286 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi175 smithi175 *:9100 running (4m) 102s ago 13m 11.1M - 1.7.0 72c9c2088986 46c4f6b7c149 2024-09-21T10:44:42.287 INFO:teuthology.orchestra.run.smithi123.stdout:osd.0 smithi123 running (70s) 11s ago 12m 239M 1936M 19.3.0-5103-g5c1cdc3c 8f4d013db793 cc40f695c3a8 2024-09-21T10:44:42.287 INFO:teuthology.orchestra.run.smithi123.stdout:osd.1 smithi123 running (20s) 11s ago 12m 11.7M 1936M 19.3.0-5103-g5c1cdc3c 8f4d013db793 1be249025c7f 2024-09-21T10:44:42.287 INFO:teuthology.orchestra.run.smithi123.stdout:osd.2 smithi123 running (11m) 11s ago 11m 485M 1936M 17.2.0 e1d6a67b021e 7db93f8ce5b5 2024-09-21T10:44:42.287 INFO:teuthology.orchestra.run.smithi123.stdout:osd.3 smithi123 running (11m) 11s ago 11m 583M 1936M 17.2.0 e1d6a67b021e c1672dea7b1f 2024-09-21T10:44:42.287 INFO:teuthology.orchestra.run.smithi123.stdout:osd.4 smithi175 running (10m) 102s ago 10m 498M 2704M 17.2.0 e1d6a67b021e 48fe728c259e 2024-09-21T10:44:42.287 INFO:teuthology.orchestra.run.smithi123.stdout:osd.5 smithi175 running (10m) 102s ago 10m 331M 2704M 17.2.0 e1d6a67b021e e270d5a21b14 2024-09-21T10:44:42.287 INFO:teuthology.orchestra.run.smithi123.stdout:osd.6 smithi175 running (10m) 102s ago 10m 517M 2704M 17.2.0 e1d6a67b021e 9bcbf018eb20 2024-09-21T10:44:42.287 INFO:teuthology.orchestra.run.smithi123.stdout:osd.7 smithi175 running (9m) 102s ago 9m 383M 2704M 17.2.0 e1d6a67b021e e9b8f1f90994 2024-09-21T10:44:42.287 INFO:teuthology.orchestra.run.smithi123.stdout:prometheus.smithi123 smithi123 *:9095 running (3m) 11s ago 13m 50.8M - 2.51.0 1d3b7f56885b 37f944c9de88 2024-09-21T10:44:42.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:42 smithi175 ceph-mon[49304]: from='client.34254 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:44:42.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:42 smithi175 ceph-mon[49304]: osdmap e102: 8 total, 8 up, 8 in 2024-09-21T10:44:42.832 INFO:teuthology.orchestra.run.smithi123.stdout: 5 190836 10.25 MB/sec execute 343 sec latency 676.647 ms 2024-09-21T10:44:42.856 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-09-21T10:44:42.856 INFO:teuthology.orchestra.run.smithi123.stdout: "mon": { 2024-09-21T10:44:42.856 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev)": 2 2024-09-21T10:44:42.856 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:44:42.856 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr": { 2024-09-21T10:44:42.856 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev)": 2 2024-09-21T10:44:42.856 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:44:42.856 INFO:teuthology.orchestra.run.smithi123.stdout: "osd": { 2024-09-21T10:44:42.856 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-09-21T10:44:42.856 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev)": 2 2024-09-21T10:44:42.856 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:44:42.856 INFO:teuthology.orchestra.run.smithi123.stdout: "mds": { 2024-09-21T10:44:42.857 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T10:44:42.857 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:44:42.857 INFO:teuthology.orchestra.run.smithi123.stdout: "overall": { 2024-09-21T10:44:42.857 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8, 2024-09-21T10:44:42.857 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev)": 6 2024-09-21T10:44:42.857 INFO:teuthology.orchestra.run.smithi123.stdout: } 2024-09-21T10:44:42.857 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-09-21T10:44:43.328 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-09-21T10:44:43.328 INFO:teuthology.orchestra.run.smithi123.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472", 2024-09-21T10:44:43.328 INFO:teuthology.orchestra.run.smithi123.stdout: "in_progress": true, 2024-09-21T10:44:43.328 INFO:teuthology.orchestra.run.smithi123.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T10:44:43.328 INFO:teuthology.orchestra.run.smithi123.stdout: "services_complete": [ 2024-09-21T10:44:43.328 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr", 2024-09-21T10:44:43.329 INFO:teuthology.orchestra.run.smithi123.stdout: "crash", 2024-09-21T10:44:43.329 INFO:teuthology.orchestra.run.smithi123.stdout: "mon" 2024-09-21T10:44:43.329 INFO:teuthology.orchestra.run.smithi123.stdout: ], 2024-09-21T10:44:43.329 INFO:teuthology.orchestra.run.smithi123.stdout: "progress": "8/23 daemons upgraded", 2024-09-21T10:44:43.329 INFO:teuthology.orchestra.run.smithi123.stdout: "message": "Currently upgrading osd daemons", 2024-09-21T10:44:43.329 INFO:teuthology.orchestra.run.smithi123.stdout: "is_paused": false 2024-09-21T10:44:43.329 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-09-21T10:44:43.619 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:43 smithi123 ceph-mon[63856]: from='client.44203 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:44:43.619 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:43 smithi123 ceph-mon[63856]: pgmap v119: 97 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 1 activating, 1 peering, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 342 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 3.9 MiB/s wr, 398 op/s; 169/1581 objects degraded (10.689%); 1.0 MiB/s, 22 objects/s recovering 2024-09-21T10:44:43.619 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:43 smithi123 ceph-mon[63856]: from='client.34260 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:44:43.620 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:43 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:44:43.620 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:43 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:44:43.620 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:43 smithi123 ceph-mon[63856]: from='client.? 172.21.15.123:0/805388572' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:44:43.620 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:43 smithi123 ceph-mon[63856]: osdmap e103: 8 total, 8 up, 8 in 2024-09-21T10:44:43.832 INFO:teuthology.orchestra.run.smithi123.stdout: 5 191160 10.24 MB/sec execute 344 sec latency 877.843 ms 2024-09-21T10:44:43.855 INFO:teuthology.orchestra.run.smithi123.stdout:HEALTH_WARN Degraded data redundancy: 169/1581 objects degraded (10.689%), 7 pgs degraded 2024-09-21T10:44:43.855 INFO:teuthology.orchestra.run.smithi123.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 169/1581 objects degraded (10.689%), 7 pgs degraded 2024-09-21T10:44:43.855 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.4 is active+recovery_wait+undersized+degraded+remapped, acting [2,5] 2024-09-21T10:44:43.855 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.8 is active+recovery_wait+undersized+degraded+remapped, acting [3,7] 2024-09-21T10:44:43.855 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.a is active+recovery_wait+undersized+degraded+remapped, acting [6,4] 2024-09-21T10:44:43.855 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.16 is active+recovering+undersized+degraded+remapped, acting [5,7] 2024-09-21T10:44:43.855 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.18 is active+recovery_wait+undersized+degraded+remapped, acting [3,0] 2024-09-21T10:44:43.855 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.1a is active+recovery_wait+undersized+degraded+remapped, acting [4,2] 2024-09-21T10:44:43.855 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.1c is active+recovery_wait+undersized+degraded+remapped, acting [5,4] 2024-09-21T10:44:43.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:43 smithi175 ceph-mon[49304]: from='client.44203 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:44:43.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:43 smithi175 ceph-mon[49304]: pgmap v119: 97 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 1 activating, 1 peering, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 342 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 3.9 MiB/s wr, 398 op/s; 169/1581 objects degraded (10.689%); 1.0 MiB/s, 22 objects/s recovering 2024-09-21T10:44:43.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:43 smithi175 ceph-mon[49304]: from='client.34260 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:44:43.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:43 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:44:43.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:43 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:44:43.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:43 smithi175 ceph-mon[49304]: from='client.? 172.21.15.123:0/805388572' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:44:43.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:43 smithi175 ceph-mon[49304]: osdmap e103: 8 total, 8 up, 8 in 2024-09-21T10:44:44.655 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:44 smithi123 ceph-mon[63856]: from='client.34268 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:44:44.655 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:44 smithi123 ceph-mon[63856]: from='client.? 172.21.15.123:0/2190218741' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T10:44:44.655 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:44 smithi123 ceph-mon[63856]: osdmap e104: 8 total, 8 up, 8 in 2024-09-21T10:44:44.832 INFO:teuthology.orchestra.run.smithi123.stdout: 5 191428 10.23 MB/sec execute 345 sec latency 38.391 ms 2024-09-21T10:44:45.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:44 smithi175 ceph-mon[49304]: from='client.34268 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:44:45.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:44 smithi175 ceph-mon[49304]: from='client.? 172.21.15.123:0/2190218741' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T10:44:45.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:44 smithi175 ceph-mon[49304]: osdmap e104: 8 total, 8 up, 8 in 2024-09-21T10:44:45.669 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:45 smithi123 ceph-mon[63856]: pgmap v122: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 peering, 92 active+clean; 345 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 5.5 MiB/s wr, 601 op/s; 65/1584 objects degraded (4.104%); 0 B/s, 28 objects/s recovering 2024-09-21T10:44:45.669 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:45 smithi123 ceph-mon[63856]: Health check update: Degraded data redundancy: 169/1581 objects degraded (10.689%), 7 pgs degraded (PG_DEGRADED) 2024-09-21T10:44:45.669 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:45 smithi123 ceph-mon[63856]: osdmap e105: 8 total, 8 up, 8 in 2024-09-21T10:44:45.832 INFO:teuthology.orchestra.run.smithi123.stdout: 5 191652 10.22 MB/sec execute 346 sec latency 817.416 ms 2024-09-21T10:44:46.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:45 smithi175 ceph-mon[49304]: pgmap v122: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 peering, 92 active+clean; 345 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 5.5 MiB/s wr, 601 op/s; 65/1584 objects degraded (4.104%); 0 B/s, 28 objects/s recovering 2024-09-21T10:44:46.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:45 smithi175 ceph-mon[49304]: Health check update: Degraded data redundancy: 169/1581 objects degraded (10.689%), 7 pgs degraded (PG_DEGRADED) 2024-09-21T10:44:46.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:45 smithi175 ceph-mon[49304]: osdmap e105: 8 total, 8 up, 8 in 2024-09-21T10:44:46.834 INFO:teuthology.orchestra.run.smithi123.stdout: 5 192166 10.23 MB/sec execute 347 sec latency 1004.692 ms 2024-09-21T10:44:47.181 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:47 smithi123 ceph-mon[63856]: osdmap e106: 8 total, 8 up, 8 in 2024-09-21T10:44:47.181 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:47 smithi123 ceph-mon[63856]: pgmap v125: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 peering, 93 active+clean; 347 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.7 MiB/s wr, 645 op/s; 65/1614 objects degraded (4.027%); 0 B/s, 28 objects/s recovering 2024-09-21T10:44:47.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:47 smithi175 ceph-mon[49304]: osdmap e106: 8 total, 8 up, 8 in 2024-09-21T10:44:47.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:47 smithi175 ceph-mon[49304]: pgmap v125: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 peering, 93 active+clean; 347 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.7 MiB/s wr, 645 op/s; 65/1614 objects degraded (4.027%); 0 B/s, 28 objects/s recovering 2024-09-21T10:44:47.834 INFO:teuthology.orchestra.run.smithi123.stdout: 5 192551 10.23 MB/sec execute 348 sec latency 656.946 ms 2024-09-21T10:44:48.196 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:48 smithi123 ceph-mon[63856]: osdmap e107: 8 total, 8 up, 8 in 2024-09-21T10:44:48.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:48 smithi175 ceph-mon[49304]: osdmap e107: 8 total, 8 up, 8 in 2024-09-21T10:44:48.834 INFO:teuthology.orchestra.run.smithi123.stdout: 5 193029 10.23 MB/sec execute 349 sec latency 858.518 ms 2024-09-21T10:44:49.200 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:49 smithi123 ceph-mon[63856]: osdmap e108: 8 total, 8 up, 8 in 2024-09-21T10:44:49.200 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:49 smithi123 ceph-mon[63856]: pgmap v128: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 peering, 94 active+clean; 347 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 1.5 MiB/s wr, 64 op/s; 33/1629 objects degraded (2.026%); 0 B/s, 7 objects/s recovering 2024-09-21T10:44:49.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:49 smithi175 ceph-mon[49304]: osdmap e108: 8 total, 8 up, 8 in 2024-09-21T10:44:49.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:49 smithi175 ceph-mon[49304]: pgmap v128: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 peering, 94 active+clean; 347 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 1.5 MiB/s wr, 64 op/s; 33/1629 objects degraded (2.026%); 0 B/s, 7 objects/s recovering 2024-09-21T10:44:49.834 INFO:teuthology.orchestra.run.smithi123.stdout: 5 193654 10.24 MB/sec execute 350 sec latency 22.706 ms 2024-09-21T10:44:50.221 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:50 smithi123 ceph-mon[63856]: Health check update: Degraded data redundancy: 33/1629 objects degraded (2.026%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T10:44:50.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:50 smithi175 ceph-mon[49304]: Health check update: Degraded data redundancy: 33/1629 objects degraded (2.026%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T10:44:50.834 INFO:teuthology.orchestra.run.smithi123.stdout: 5 194233 10.28 MB/sec execute 351 sec latency 30.281 ms 2024-09-21T10:44:51.232 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:51 smithi123 ceph-mon[63856]: pgmap v129: 97 pgs: 97 active+clean; 348 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 5.4 MiB/s wr, 498 op/s; 0 B/s, 14 objects/s recovering 2024-09-21T10:44:51.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:51 smithi175 ceph-mon[49304]: pgmap v129: 97 pgs: 97 active+clean; 348 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 5.4 MiB/s wr, 498 op/s; 0 B/s, 14 objects/s recovering 2024-09-21T10:44:51.835 INFO:teuthology.orchestra.run.smithi123.stdout: 5 194754 10.29 MB/sec execute 352 sec latency 35.505 ms 2024-09-21T10:44:52.239 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:52 smithi123 ceph-mon[63856]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 33/1629 objects degraded (2.026%), 2 pgs degraded) 2024-09-21T10:44:52.239 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:52 smithi123 ceph-mon[63856]: Cluster is now healthy 2024-09-21T10:44:52.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:52 smithi175 ceph-mon[49304]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 33/1629 objects degraded (2.026%), 2 pgs degraded) 2024-09-21T10:44:52.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:52 smithi175 ceph-mon[49304]: Cluster is now healthy 2024-09-21T10:44:52.834 INFO:teuthology.orchestra.run.smithi123.stdout: 5 195105 10.28 MB/sec execute 353 sec latency 33.375 ms 2024-09-21T10:44:53.251 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:53 smithi123 ceph-mon[63856]: pgmap v130: 97 pgs: 97 active+clean; 348 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.1 KiB/s rd, 4.5 MiB/s wr, 418 op/s; 0 B/s, 12 objects/s recovering 2024-09-21T10:44:53.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:53 smithi175 ceph-mon[49304]: pgmap v130: 97 pgs: 97 active+clean; 348 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.1 KiB/s rd, 4.5 MiB/s wr, 418 op/s; 0 B/s, 12 objects/s recovering 2024-09-21T10:44:53.835 INFO:teuthology.orchestra.run.smithi123.stdout: 5 195498 10.28 MB/sec execute 354 sec latency 42.609 ms 2024-09-21T10:44:54.274 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:54 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-21T10:44:54.274 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:54 smithi123 ceph-mon[63856]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-21T10:44:54.274 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:54 smithi123 ceph-mon[63856]: Upgrade: osd.2 is safe to restart 2024-09-21T10:44:54.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:54 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-21T10:44:54.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:54 smithi175 ceph-mon[49304]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-21T10:44:54.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:54 smithi175 ceph-mon[49304]: Upgrade: osd.2 is safe to restart 2024-09-21T10:44:54.835 INFO:teuthology.orchestra.run.smithi123.stdout: 5 195894 10.29 MB/sec execute 355 sec latency 49.342 ms 2024-09-21T10:44:55.286 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:55 smithi123 ceph-mon[63856]: pgmap v131: 97 pgs: 97 active+clean; 358 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.9 MiB/s wr, 625 op/s; 0 B/s, 9 objects/s recovering 2024-09-21T10:44:55.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:55 smithi175 ceph-mon[49304]: pgmap v131: 97 pgs: 97 active+clean; 358 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.9 MiB/s wr, 625 op/s; 0 B/s, 9 objects/s recovering 2024-09-21T10:44:55.835 INFO:teuthology.orchestra.run.smithi123.stdout: 5 196195 10.28 MB/sec execute 356 sec latency 44.075 ms 2024-09-21T10:44:56.558 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:56 smithi123 ceph-mon[63856]: Upgrade: Updating osd.2 2024-09-21T10:44:56.558 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:56 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:44:56.559 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:56 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-21T10:44:56.559 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:56 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:44:56.559 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:56 smithi123 ceph-mon[63856]: Deploying daemon osd.2 on smithi123 2024-09-21T10:44:56.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:56 smithi175 ceph-mon[49304]: Upgrade: Updating osd.2 2024-09-21T10:44:56.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:56 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:44:56.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:56 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-21T10:44:56.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:56 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:44:56.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:56 smithi175 ceph-mon[49304]: Deploying daemon osd.2 on smithi123 2024-09-21T10:44:56.836 INFO:teuthology.orchestra.run.smithi123.stdout: 5 196448 10.27 MB/sec execute 357 sec latency 90.972 ms 2024-09-21T10:44:57.569 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:57 smithi123 ceph-mon[63856]: pgmap v132: 97 pgs: 97 active+clean; 358 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 7.0 MiB/s wr, 680 op/s; 0 B/s, 8 objects/s recovering 2024-09-21T10:44:57.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:57 smithi175 ceph-mon[49304]: pgmap v132: 97 pgs: 97 active+clean; 358 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 7.0 MiB/s wr, 680 op/s; 0 B/s, 8 objects/s recovering 2024-09-21T10:44:57.836 INFO:teuthology.orchestra.run.smithi123.stdout: 5 196775 10.26 MB/sec execute 358 sec latency 67.588 ms 2024-09-21T10:44:58.836 INFO:teuthology.orchestra.run.smithi123.stdout: 5 197002 10.24 MB/sec execute 359 sec latency 51.833 ms 2024-09-21T10:44:58.848 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:58 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:44:58.848 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:58 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:44:58.848 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:58 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:44:59.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:58 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:44:59.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:58 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:44:59.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:58 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:44:59.836 INFO:teuthology.orchestra.run.smithi123.stdout: 5 197280 10.23 MB/sec execute 360 sec latency 44.159 ms 2024-09-21T10:44:59.856 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:44:59 smithi123 ceph-mon[63856]: pgmap v133: 97 pgs: 97 active+clean; 357 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 6.4 MiB/s wr, 620 op/s; 0 B/s, 7 objects/s recovering 2024-09-21T10:44:59.856 INFO:journalctl@ceph.osd.2.smithi123.stdout:Sep 21 10:44:59 smithi123 systemd[1]: Stopping Ceph osd.2 for 3ccd15e6-7804-11ef-baf6-efdc52797490... 2024-09-21T10:45:00.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:44:59 smithi175 ceph-mon[49304]: pgmap v133: 97 pgs: 97 active+clean; 357 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 6.4 MiB/s wr, 620 op/s; 0 B/s, 7 objects/s recovering 2024-09-21T10:45:00.143 INFO:journalctl@ceph.osd.2.smithi123.stdout:Sep 21 10:45:00 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-2[41889]: 2024-09-21T10:45:00.070+0000 7eff1074f700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:45:00.143 INFO:journalctl@ceph.osd.2.smithi123.stdout:Sep 21 10:45:00 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-2[41889]: 2024-09-21T10:45:00.070+0000 7eff1074f700 -1 osd.2 108 *** Got signal Terminated *** 2024-09-21T10:45:00.143 INFO:journalctl@ceph.osd.2.smithi123.stdout:Sep 21 10:45:00 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-2[41889]: 2024-09-21T10:45:00.070+0000 7eff1074f700 -1 osd.2 108 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T10:45:00.836 INFO:teuthology.orchestra.run.smithi123.stdout: 5 197553 10.24 MB/sec execute 361 sec latency 63.273 ms 2024-09-21T10:45:01.172 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:01 smithi123 ceph-mon[63856]: osd.2 marked itself down and dead 2024-09-21T10:45:01.172 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:01 smithi123 ceph-mon[63856]: pgmap v134: 97 pgs: 97 active+clean; 347 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 7.1 MiB/s wr, 714 op/s; 0 B/s, 3 objects/s recovering 2024-09-21T10:45:01.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:01 smithi175 ceph-mon[49304]: osd.2 marked itself down and dead 2024-09-21T10:45:01.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:01 smithi175 ceph-mon[49304]: pgmap v134: 97 pgs: 97 active+clean; 347 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 7.1 MiB/s wr, 714 op/s; 0 B/s, 3 objects/s recovering 2024-09-21T10:45:01.836 INFO:teuthology.orchestra.run.smithi123.stdout: 5 197731 10.22 MB/sec execute 362 sec latency 789.301 ms 2024-09-21T10:45:01.851 INFO:journalctl@ceph.osd.2.smithi123.stdout:Sep 21 10:45:01 smithi123 podman[73087]: 2024-09-21 10:45:01.679715297 +0000 UTC m=+1.732714504 container died 7db93f8ce5b5b6e9d7e53566f7e5b2f056f0cb5abde1f2541297da1c9ef688c1 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-2, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, release=754, io.openshift.expose-services=, version=8, name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, io.k8s.display-name=CentOS Stream 8) 2024-09-21T10:45:02.204 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:02 smithi123 ceph-mon[63856]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T10:45:02.204 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:02 smithi123 ceph-mon[63856]: osdmap e109: 8 total, 7 up, 8 in 2024-09-21T10:45:02.204 INFO:journalctl@ceph.osd.2.smithi123.stdout:Sep 21 10:45:02 smithi123 podman[73087]: 2024-09-21 10:45:02.160067393 +0000 UTC m=+2.213066600 container cleanup 7db93f8ce5b5b6e9d7e53566f7e5b2f056f0cb5abde1f2541297da1c9ef688c1 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-2, distribution-scope=public, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, release=754, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, version=8, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-09-21T10:45:02.204 INFO:journalctl@ceph.osd.2.smithi123.stdout:Sep 21 10:45:02 smithi123 bash[73087]: ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-2 2024-09-21T10:45:02.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:02 smithi175 ceph-mon[49304]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T10:45:02.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:02 smithi175 ceph-mon[49304]: osdmap e109: 8 total, 7 up, 8 in 2024-09-21T10:45:02.455 INFO:journalctl@ceph.osd.2.smithi123.stdout:Sep 21 10:45:02 smithi123 podman[73105]: 2024-09-21 10:45:02.429036391 +0000 UTC m=+0.752501184 container remove 7db93f8ce5b5b6e9d7e53566f7e5b2f056f0cb5abde1f2541297da1c9ef688c1 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-2, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , release=754, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, vcs-type=git, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD) 2024-09-21T10:45:02.836 INFO:teuthology.orchestra.run.smithi123.stdout: 5 197993 10.22 MB/sec execute 363 sec latency 985.951 ms 2024-09-21T10:45:02.968 INFO:journalctl@ceph.osd.2.smithi123.stdout:Sep 21 10:45:02 smithi123 podman[73189]: 2024-09-21 10:45:02.714534848 +0000 UTC m=+0.017964243 image pull 8f4d013db793795fef09ff5edf96e6b42ba8cd2c9158b2f0b7a5d7c1e657c063 quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472 2024-09-21T10:45:02.968 INFO:journalctl@ceph.osd.2.smithi123.stdout:Sep 21 10:45:02 smithi123 podman[73189]: 2024-09-21 10:45:02.87180034 +0000 UTC m=+0.175229734 container create a5a8dd234d3d3bd4b41537d783a5206e3c31741ab37b1e4832263f2da4b641b1 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-2-deactivate, io.buildah.version=1.37.2, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3) 2024-09-21T10:45:03.222 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:03 smithi123 ceph-mon[63856]: osdmap e110: 8 total, 7 up, 8 in 2024-09-21T10:45:03.222 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:03 smithi123 ceph-mon[63856]: pgmap v137: 97 pgs: 6 stale+active+clean, 91 active+clean; 347 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.9 MiB/s wr, 456 op/s 2024-09-21T10:45:03.222 INFO:journalctl@ceph.osd.2.smithi123.stdout:Sep 21 10:45:03 smithi123 podman[73189]: 2024-09-21 10:45:03.174951826 +0000 UTC m=+0.478381223 container init a5a8dd234d3d3bd4b41537d783a5206e3c31741ab37b1e4832263f2da4b641b1 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-2-deactivate, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-09-21T10:45:03.223 INFO:journalctl@ceph.osd.2.smithi123.stdout:Sep 21 10:45:03 smithi123 podman[73189]: 2024-09-21 10:45:03.183721522 +0000 UTC m=+0.487150909 container start a5a8dd234d3d3bd4b41537d783a5206e3c31741ab37b1e4832263f2da4b641b1 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-2-deactivate, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T10:45:03.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:03 smithi175 ceph-mon[49304]: osdmap e110: 8 total, 7 up, 8 in 2024-09-21T10:45:03.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:03 smithi175 ceph-mon[49304]: pgmap v137: 97 pgs: 6 stale+active+clean, 91 active+clean; 347 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.9 MiB/s wr, 456 op/s 2024-09-21T10:45:03.473 INFO:journalctl@ceph.osd.2.smithi123.stdout:Sep 21 10:45:03 smithi123 podman[73189]: 2024-09-21 10:45:03.316739787 +0000 UTC m=+0.620169180 container attach a5a8dd234d3d3bd4b41537d783a5206e3c31741ab37b1e4832263f2da4b641b1 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-2-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T10:45:03.473 INFO:journalctl@ceph.osd.2.smithi123.stdout:Sep 21 10:45:03 smithi123 podman[73189]: 2024-09-21 10:45:03.458401461 +0000 UTC m=+0.761830856 container died a5a8dd234d3d3bd4b41537d783a5206e3c31741ab37b1e4832263f2da4b641b1 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-2-deactivate, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-09-21T10:45:03.836 INFO:teuthology.orchestra.run.smithi123.stdout: 5 198315 10.21 MB/sec execute 364 sec latency 72.973 ms 2024-09-21T10:45:04.485 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:04 smithi123 ceph-mon[63856]: Health check failed: Degraded data redundancy: 139/1605 objects degraded (8.660%), 19 pgs degraded (PG_DEGRADED) 2024-09-21T10:45:04.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:04 smithi175 ceph-mon[49304]: Health check failed: Degraded data redundancy: 139/1605 objects degraded (8.660%), 19 pgs degraded (PG_DEGRADED) 2024-09-21T10:45:04.836 INFO:teuthology.orchestra.run.smithi123.stdout: 5 198965 10.23 MB/sec execute 365 sec latency 31.532 ms 2024-09-21T10:45:04.987 INFO:journalctl@ceph.osd.2.smithi123.stdout:Sep 21 10:45:04 smithi123 podman[73211]: 2024-09-21 10:45:04.909301563 +0000 UTC m=+1.444723601 container remove a5a8dd234d3d3bd4b41537d783a5206e3c31741ab37b1e4832263f2da4b641b1 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-2-deactivate, ceph=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0) 2024-09-21T10:45:04.987 INFO:journalctl@ceph.osd.2.smithi123.stdout:Sep 21 10:45:04 smithi123 systemd[1]: ceph-3ccd15e6-7804-11ef-baf6-efdc52797490@osd.2.service: Deactivated successfully. 2024-09-21T10:45:04.987 INFO:journalctl@ceph.osd.2.smithi123.stdout:Sep 21 10:45:04 smithi123 systemd[1]: Stopped Ceph osd.2 for 3ccd15e6-7804-11ef-baf6-efdc52797490. 2024-09-21T10:45:04.987 INFO:journalctl@ceph.osd.2.smithi123.stdout:Sep 21 10:45:04 smithi123 systemd[1]: ceph-3ccd15e6-7804-11ef-baf6-efdc52797490@osd.2.service: Consumed 2min 48.735s CPU time. 2024-09-21T10:45:05.489 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:05 smithi123 ceph-mon[63856]: pgmap v138: 97 pgs: 6 active+undersized, 19 active+undersized+degraded, 72 active+clean; 343 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.5 MiB/s wr, 505 op/s; 139/1605 objects degraded (8.660%) 2024-09-21T10:45:05.489 INFO:journalctl@ceph.osd.2.smithi123.stdout:Sep 21 10:45:05 smithi123 systemd[1]: Starting Ceph osd.2 for 3ccd15e6-7804-11ef-baf6-efdc52797490... 2024-09-21T10:45:05.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:05 smithi175 ceph-mon[49304]: pgmap v138: 97 pgs: 6 active+undersized, 19 active+undersized+degraded, 72 active+clean; 343 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.5 MiB/s wr, 505 op/s; 139/1605 objects degraded (8.660%) 2024-09-21T10:45:05.740 INFO:journalctl@ceph.osd.2.smithi123.stdout:Sep 21 10:45:05 smithi123 podman[73307]: 2024-09-21 10:45:05.616347384 +0000 UTC m=+0.026064792 image pull 8f4d013db793795fef09ff5edf96e6b42ba8cd2c9158b2f0b7a5d7c1e657c063 quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472 2024-09-21T10:45:05.836 INFO:teuthology.orchestra.run.smithi123.stdout: 5 199640 10.25 MB/sec execute 366 sec latency 23.264 ms 2024-09-21T10:45:05.991 INFO:journalctl@ceph.osd.2.smithi123.stdout:Sep 21 10:45:05 smithi123 podman[73307]: 2024-09-21 10:45:05.769158341 +0000 UTC m=+0.178875750 container create 8634c6e1b17d5fa3de73ce1522a408587c53bbb5b28dd6ed1719412d1527dd86 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-2-activate, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731) 2024-09-21T10:45:06.243 INFO:journalctl@ceph.osd.2.smithi123.stdout:Sep 21 10:45:06 smithi123 podman[73307]: 2024-09-21 10:45:06.190132429 +0000 UTC m=+0.599849846 container init 8634c6e1b17d5fa3de73ce1522a408587c53bbb5b28dd6ed1719412d1527dd86 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-2-activate, ceph=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.37.2) 2024-09-21T10:45:06.243 INFO:journalctl@ceph.osd.2.smithi123.stdout:Sep 21 10:45:06 smithi123 podman[73307]: 2024-09-21 10:45:06.19959959 +0000 UTC m=+0.609316993 container start 8634c6e1b17d5fa3de73ce1522a408587c53bbb5b28dd6ed1719412d1527dd86 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-2-activate, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T10:45:06.494 INFO:journalctl@ceph.osd.2.smithi123.stdout:Sep 21 10:45:06 smithi123 podman[73307]: 2024-09-21 10:45:06.324006595 +0000 UTC m=+0.733724009 container attach 8634c6e1b17d5fa3de73ce1522a408587c53bbb5b28dd6ed1719412d1527dd86 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-2-activate, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3) 2024-09-21T10:45:06.494 INFO:journalctl@ceph.osd.2.smithi123.stdout:Sep 21 10:45:06 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-2-activate[73321]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:45:06.495 INFO:journalctl@ceph.osd.2.smithi123.stdout:Sep 21 10:45:06 smithi123 bash[73307]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:45:06.495 INFO:journalctl@ceph.osd.2.smithi123.stdout:Sep 21 10:45:06 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-2-activate[73321]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:45:06.495 INFO:journalctl@ceph.osd.2.smithi123.stdout:Sep 21 10:45:06 smithi123 bash[73307]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:45:06.836 INFO:teuthology.orchestra.run.smithi123.stdout: 5 200389 10.27 MB/sec execute 367 sec latency 38.251 ms 2024-09-21T10:45:07.503 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:07 smithi123 ceph-mon[63856]: pgmap v139: 97 pgs: 7 active+undersized, 20 active+undersized+degraded, 70 active+clean; 344 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.0 MiB/s wr, 517 op/s; 152/1620 objects degraded (9.383%) 2024-09-21T10:45:07.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:07 smithi175 ceph-mon[49304]: pgmap v139: 97 pgs: 7 active+undersized, 20 active+undersized+degraded, 70 active+clean; 344 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.0 MiB/s wr, 517 op/s; 152/1620 objects degraded (9.383%) 2024-09-21T10:45:07.836 INFO:teuthology.orchestra.run.smithi123.stdout: 5 200869 10.27 MB/sec execute 368 sec latency 40.324 ms 2024-09-21T10:45:08.836 INFO:teuthology.orchestra.run.smithi123.stdout: 5 201410 10.30 MB/sec execute 369 sec latency 46.798 ms 2024-09-21T10:45:09.591 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:09 smithi123 ceph-mon[63856]: pgmap v140: 97 pgs: 7 active+undersized, 20 active+undersized+degraded, 70 active+clean; 344 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 3.0 KiB/s rd, 3.1 MiB/s wr, 268 op/s; 152/1626 objects degraded (9.348%) 2024-09-21T10:45:09.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:09 smithi175 ceph-mon[49304]: pgmap v140: 97 pgs: 7 active+undersized, 20 active+undersized+degraded, 70 active+clean; 344 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 3.0 KiB/s rd, 3.1 MiB/s wr, 268 op/s; 152/1626 objects degraded (9.348%) 2024-09-21T10:45:09.836 INFO:teuthology.orchestra.run.smithi123.stdout: 5 201906 10.30 MB/sec execute 370 sec latency 63.736 ms 2024-09-21T10:45:10.612 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:10 smithi123 ceph-mon[63856]: Health check update: Degraded data redundancy: 152/1626 objects degraded (9.348%), 20 pgs degraded (PG_DEGRADED) 2024-09-21T10:45:10.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:10 smithi175 ceph-mon[49304]: Health check update: Degraded data redundancy: 152/1626 objects degraded (9.348%), 20 pgs degraded (PG_DEGRADED) 2024-09-21T10:45:10.836 INFO:teuthology.orchestra.run.smithi123.stdout: 5 202444 10.32 MB/sec execute 371 sec latency 32.641 ms 2024-09-21T10:45:11.116 INFO:journalctl@ceph.osd.2.smithi123.stdout:Sep 21 10:45:11 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-2-activate[73321]: --> Failed to activate via raw: did not find any matching OSD to activate 2024-09-21T10:45:11.116 INFO:journalctl@ceph.osd.2.smithi123.stdout:Sep 21 10:45:11 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-2-activate[73321]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:45:11.116 INFO:journalctl@ceph.osd.2.smithi123.stdout:Sep 21 10:45:11 smithi123 bash[73307]: --> Failed to activate via raw: did not find any matching OSD to activate 2024-09-21T10:45:11.116 INFO:journalctl@ceph.osd.2.smithi123.stdout:Sep 21 10:45:11 smithi123 bash[73307]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:45:11.117 INFO:journalctl@ceph.osd.2.smithi123.stdout:Sep 21 10:45:11 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-2-activate[73321]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:45:11.117 INFO:journalctl@ceph.osd.2.smithi123.stdout:Sep 21 10:45:11 smithi123 bash[73307]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:45:11.373 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:11 smithi123 ceph-mon[63856]: pgmap v141: 97 pgs: 7 active+undersized, 20 active+undersized+degraded, 70 active+clean; 352 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 7.0 MiB/s wr, 671 op/s; 154/1695 objects degraded (9.086%) 2024-09-21T10:45:11.373 INFO:journalctl@ceph.osd.2.smithi123.stdout:Sep 21 10:45:11 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-2-activate[73321]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-21T10:45:11.373 INFO:journalctl@ceph.osd.2.smithi123.stdout:Sep 21 10:45:11 smithi123 bash[73307]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-21T10:45:11.374 INFO:journalctl@ceph.osd.2.smithi123.stdout:Sep 21 10:45:11 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-2-activate[73321]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-2 --no-mon-config 2024-09-21T10:45:11.374 INFO:journalctl@ceph.osd.2.smithi123.stdout:Sep 21 10:45:11 smithi123 bash[73307]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-2 --no-mon-config 2024-09-21T10:45:11.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:11 smithi175 ceph-mon[49304]: pgmap v141: 97 pgs: 7 active+undersized, 20 active+undersized+degraded, 70 active+clean; 352 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 7.0 MiB/s wr, 671 op/s; 154/1695 objects degraded (9.086%) 2024-09-21T10:45:11.624 INFO:journalctl@ceph.osd.2.smithi123.stdout:Sep 21 10:45:11 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-2-activate[73321]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-2/block 2024-09-21T10:45:11.624 INFO:journalctl@ceph.osd.2.smithi123.stdout:Sep 21 10:45:11 smithi123 bash[73307]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-2/block 2024-09-21T10:45:11.624 INFO:journalctl@ceph.osd.2.smithi123.stdout:Sep 21 10:45:11 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-2-activate[73321]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-2/block 2024-09-21T10:45:11.624 INFO:journalctl@ceph.osd.2.smithi123.stdout:Sep 21 10:45:11 smithi123 bash[73307]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-2/block 2024-09-21T10:45:11.624 INFO:journalctl@ceph.osd.2.smithi123.stdout:Sep 21 10:45:11 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-2-activate[73321]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-21T10:45:11.625 INFO:journalctl@ceph.osd.2.smithi123.stdout:Sep 21 10:45:11 smithi123 bash[73307]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-21T10:45:11.625 INFO:journalctl@ceph.osd.2.smithi123.stdout:Sep 21 10:45:11 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-2-activate[73321]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-21T10:45:11.625 INFO:journalctl@ceph.osd.2.smithi123.stdout:Sep 21 10:45:11 smithi123 bash[73307]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-21T10:45:11.625 INFO:journalctl@ceph.osd.2.smithi123.stdout:Sep 21 10:45:11 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-2-activate[73321]: --> ceph-volume lvm activate successful for osd ID: 2 2024-09-21T10:45:11.625 INFO:journalctl@ceph.osd.2.smithi123.stdout:Sep 21 10:45:11 smithi123 bash[73307]: --> ceph-volume lvm activate successful for osd ID: 2 2024-09-21T10:45:11.836 INFO:teuthology.orchestra.run.smithi123.stdout: 5 202988 10.33 MB/sec execute 372 sec latency 34.819 ms 2024-09-21T10:45:11.878 INFO:journalctl@ceph.osd.2.smithi123.stdout:Sep 21 10:45:11 smithi123 podman[73307]: 2024-09-21 10:45:11.634825355 +0000 UTC m=+6.044542764 container died 8634c6e1b17d5fa3de73ce1522a408587c53bbb5b28dd6ed1719412d1527dd86 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-2-activate, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=) 2024-09-21T10:45:12.836 INFO:teuthology.orchestra.run.smithi123.stdout: 5 203449 10.34 MB/sec execute 373 sec latency 31.814 ms 2024-09-21T10:45:13.168 INFO:journalctl@ceph.osd.2.smithi123.stdout:Sep 21 10:45:12 smithi123 podman[73538]: 2024-09-21 10:45:12.984211401 +0000 UTC m=+1.339966663 container remove 8634c6e1b17d5fa3de73ce1522a408587c53bbb5b28dd6ed1719412d1527dd86 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-2-activate, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-09-21T10:45:13.424 INFO:journalctl@ceph.osd.2.smithi123.stdout:Sep 21 10:45:13 smithi123 podman[73609]: 2024-09-21 10:45:13.160332321 +0000 UTC m=+0.037345202 image pull 8f4d013db793795fef09ff5edf96e6b42ba8cd2c9158b2f0b7a5d7c1e657c063 quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472 2024-09-21T10:45:13.424 INFO:journalctl@ceph.osd.2.smithi123.stdout:Sep 21 10:45:13 smithi123 podman[73609]: 2024-09-21 10:45:13.302429364 +0000 UTC m=+0.179442232 container create 647aec759e3c2f1473b2fea786ab91eee17aff7e2f503e41018d8b226c11ffd2 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, io.buildah.version=1.37.2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3) 2024-09-21T10:45:13.675 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:13 smithi123 ceph-mon[63856]: pgmap v142: 97 pgs: 7 active+undersized, 20 active+undersized+degraded, 70 active+clean; 352 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 6.3 MiB/s wr, 605 op/s; 154/1695 objects degraded (9.086%) 2024-09-21T10:45:13.675 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:13 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:45:13.675 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:13 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:45:13.676 INFO:journalctl@ceph.osd.2.smithi123.stdout:Sep 21 10:45:13 smithi123 podman[73609]: 2024-09-21 10:45:13.639209315 +0000 UTC m=+0.516222211 container init 647aec759e3c2f1473b2fea786ab91eee17aff7e2f503e41018d8b226c11ffd2 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-2, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, org.label-schema.schema-version=1.0) 2024-09-21T10:45:13.676 INFO:journalctl@ceph.osd.2.smithi123.stdout:Sep 21 10:45:13 smithi123 podman[73609]: 2024-09-21 10:45:13.646879546 +0000 UTC m=+0.523892414 container start 647aec759e3c2f1473b2fea786ab91eee17aff7e2f503e41018d8b226c11ffd2 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-2, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-09-21T10:45:13.836 INFO:teuthology.orchestra.run.smithi123.stdout: 5 203977 10.33 MB/sec execute 374 sec latency 52.194 ms 2024-09-21T10:45:13.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:13 smithi175 ceph-mon[49304]: pgmap v142: 97 pgs: 7 active+undersized, 20 active+undersized+degraded, 70 active+clean; 352 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 6.3 MiB/s wr, 605 op/s; 154/1695 objects degraded (9.086%) 2024-09-21T10:45:13.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:13 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:45:13.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:13 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:45:13.930 INFO:journalctl@ceph.osd.2.smithi123.stdout:Sep 21 10:45:13 smithi123 ceph-osd[73628]: -- 172.21.15.123:0/1721003304 <== mon.0 v2:172.21.15.123:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55a7b2d1c1e0 con 0x55a7b2d18000 2024-09-21T10:45:14.181 INFO:journalctl@ceph.osd.2.smithi123.stdout:Sep 21 10:45:14 smithi123 bash[73609]: 647aec759e3c2f1473b2fea786ab91eee17aff7e2f503e41018d8b226c11ffd2 2024-09-21T10:45:14.181 INFO:journalctl@ceph.osd.2.smithi123.stdout:Sep 21 10:45:14 smithi123 systemd[1]: Started Ceph osd.2 for 3ccd15e6-7804-11ef-baf6-efdc52797490. 2024-09-21T10:45:14.432 INFO:teuthology.orchestra.run.smithi123.stdout:true 2024-09-21T10:45:14.836 INFO:teuthology.orchestra.run.smithi123.stdout: 5 204395 10.34 MB/sec execute 375 sec latency 33.049 ms 2024-09-21T10:45:15.188 INFO:journalctl@ceph.osd.2.smithi123.stdout:Sep 21 10:45:15 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-2[73624]: 2024-09-21T10:45:15.056+0000 7fb594ec0640 -1 Falling back to public interface 2024-09-21T10:45:15.455 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:15 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:45:15.455 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:15 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:45:15.455 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:15 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:45:15.455 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:15 smithi123 ceph-mon[63856]: pgmap v143: 97 pgs: 7 active+undersized, 20 active+undersized+degraded, 70 active+clean; 346 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.9 MiB/s wr, 803 op/s; 151/1662 objects degraded (9.085%) 2024-09-21T10:45:15.455 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:15 smithi123 ceph-mon[63856]: from='client.34278 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:45:15.455 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:15 smithi123 ceph-mon[63856]: Health check update: Degraded data redundancy: 154/1695 objects degraded (9.086%), 20 pgs degraded (PG_DEGRADED) 2024-09-21T10:45:15.492 INFO:teuthology.orchestra.run.smithi123.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T10:45:15.492 INFO:teuthology.orchestra.run.smithi123.stdout:alertmanager.smithi123 smithi123 *:9093,9094 running (5m) 44s ago 14m 22.0M - 0.27.0 11f11916f8cd 5d550bd97de5 2024-09-21T10:45:15.492 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi123 smithi123 running (2m) 44s ago 14m 6899k - 19.3.0-5103-g5c1cdc3c 8f4d013db793 c5e1ef39a0a8 2024-09-21T10:45:15.492 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi175 smithi175 running (2m) 2m ago 14m 6895k - 19.3.0-5103-g5c1cdc3c 8f4d013db793 fb1ca999ccfd 2024-09-21T10:45:15.492 INFO:teuthology.orchestra.run.smithi123.stdout:grafana.smithi123 smithi123 *:3000 running (5m) 44s ago 14m 45.3M - dad864ee21e9 771df81763ba 2024-09-21T10:45:15.492 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi123.uqoyow smithi123 running (8m) 44s ago 8m 41.3M - 17.2.0 e1d6a67b021e f0a21eaac266 2024-09-21T10:45:15.492 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi175.jrlevw smithi175 running (8m) 2m ago 8m 13.5M - 17.2.0 e1d6a67b021e b21b16f4bc50 2024-09-21T10:45:15.492 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi123.stnuit smithi123 *:8443,9283,8765 running (4m) 44s ago 16m 517M - 19.3.0-5103-g5c1cdc3c 8f4d013db793 6812cc5cb56b 2024-09-21T10:45:15.492 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi175.txvwrq smithi175 *:8443,9283,8765 running (3m) 2m ago 14m 453M - 19.3.0-5103-g5c1cdc3c 8f4d013db793 304f0408075e 2024-09-21T10:45:15.492 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi123 smithi123 running (3m) 44s ago 16m 48.7M 2048M 19.3.0-5103-g5c1cdc3c 8f4d013db793 6b95207a2d80 2024-09-21T10:45:15.492 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi175 smithi175 running (3m) 2m ago 14m 36.7M 2048M 19.3.0-5103-g5c1cdc3c 8f4d013db793 3abee2d37f0f 2024-09-21T10:45:15.493 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.0.0.smithi123.ddatpr smithi123 *:2049 running (8m) 44s ago 8m 279M - 4.0 e1d6a67b021e 36f8cb2281eb 2024-09-21T10:45:15.493 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.1.0.smithi175.wpawer smithi175 *:2049 running (8m) 2m ago 8m 90.3M - 4.0 e1d6a67b021e 2200aeed9200 2024-09-21T10:45:15.493 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi123 smithi123 *:9100 running (5m) 44s ago 14m 10.3M - 1.7.0 72c9c2088986 ca7d0b29af81 2024-09-21T10:45:15.493 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi175 smithi175 *:9100 running (4m) 2m ago 13m 11.1M - 1.7.0 72c9c2088986 46c4f6b7c149 2024-09-21T10:45:15.493 INFO:teuthology.orchestra.run.smithi123.stdout:osd.0 smithi123 running (103s) 44s ago 13m 239M 1936M 19.3.0-5103-g5c1cdc3c 8f4d013db793 cc40f695c3a8 2024-09-21T10:45:15.493 INFO:teuthology.orchestra.run.smithi123.stdout:osd.1 smithi123 running (53s) 44s ago 12m 11.7M 1936M 19.3.0-5103-g5c1cdc3c 8f4d013db793 1be249025c7f 2024-09-21T10:45:15.493 INFO:teuthology.orchestra.run.smithi123.stdout:osd.2 smithi123 starting - - - 1936M 2024-09-21T10:45:15.493 INFO:teuthology.orchestra.run.smithi123.stdout:osd.3 smithi123 running (11m) 44s ago 11m 583M 1936M 17.2.0 e1d6a67b021e c1672dea7b1f 2024-09-21T10:45:15.493 INFO:teuthology.orchestra.run.smithi123.stdout:osd.4 smithi175 running (11m) 2m ago 11m 498M 2704M 17.2.0 e1d6a67b021e 48fe728c259e 2024-09-21T10:45:15.493 INFO:teuthology.orchestra.run.smithi123.stdout:osd.5 smithi175 running (11m) 2m ago 11m 331M 2704M 17.2.0 e1d6a67b021e e270d5a21b14 2024-09-21T10:45:15.493 INFO:teuthology.orchestra.run.smithi123.stdout:osd.6 smithi175 running (10m) 2m ago 10m 517M 2704M 17.2.0 e1d6a67b021e 9bcbf018eb20 2024-09-21T10:45:15.493 INFO:teuthology.orchestra.run.smithi123.stdout:osd.7 smithi175 running (10m) 2m ago 10m 383M 2704M 17.2.0 e1d6a67b021e e9b8f1f90994 2024-09-21T10:45:15.493 INFO:teuthology.orchestra.run.smithi123.stdout:prometheus.smithi123 smithi123 *:9095 running (4m) 44s ago 13m 50.8M - 2.51.0 1d3b7f56885b 37f944c9de88 2024-09-21T10:45:15.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:15 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:45:15.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:15 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:45:15.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:15 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:45:15.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:15 smithi175 ceph-mon[49304]: pgmap v143: 97 pgs: 7 active+undersized, 20 active+undersized+degraded, 70 active+clean; 346 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.9 MiB/s wr, 803 op/s; 151/1662 objects degraded (9.085%) 2024-09-21T10:45:15.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:15 smithi175 ceph-mon[49304]: from='client.34278 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:45:15.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:15 smithi175 ceph-mon[49304]: Health check update: Degraded data redundancy: 154/1695 objects degraded (9.086%), 20 pgs degraded (PG_DEGRADED) 2024-09-21T10:45:15.836 INFO:teuthology.orchestra.run.smithi123.stdout: 5 204829 10.35 MB/sec execute 376 sec latency 45.607 ms 2024-09-21T10:45:16.069 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-09-21T10:45:16.070 INFO:teuthology.orchestra.run.smithi123.stdout: "mon": { 2024-09-21T10:45:16.070 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev)": 2 2024-09-21T10:45:16.070 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:45:16.070 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr": { 2024-09-21T10:45:16.070 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev)": 2 2024-09-21T10:45:16.070 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:45:16.070 INFO:teuthology.orchestra.run.smithi123.stdout: "osd": { 2024-09-21T10:45:16.070 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-09-21T10:45:16.070 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev)": 2 2024-09-21T10:45:16.070 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:45:16.070 INFO:teuthology.orchestra.run.smithi123.stdout: "mds": { 2024-09-21T10:45:16.070 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T10:45:16.071 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:45:16.071 INFO:teuthology.orchestra.run.smithi123.stdout: "overall": { 2024-09-21T10:45:16.071 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-09-21T10:45:16.071 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev)": 6 2024-09-21T10:45:16.071 INFO:teuthology.orchestra.run.smithi123.stdout: } 2024-09-21T10:45:16.071 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-09-21T10:45:16.461 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:16 smithi123 ceph-mon[63856]: from='client.44227 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:45:16.461 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:16 smithi123 ceph-mon[63856]: from='client.44231 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:45:16.461 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:16 smithi123 ceph-mon[63856]: from='client.? 172.21.15.123:0/2764912128' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:45:16.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:16 smithi175 ceph-mon[49304]: from='client.44227 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:45:16.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:16 smithi175 ceph-mon[49304]: from='client.44231 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:45:16.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:16 smithi175 ceph-mon[49304]: from='client.? 172.21.15.123:0/2764912128' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:45:16.702 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-09-21T10:45:16.702 INFO:teuthology.orchestra.run.smithi123.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472", 2024-09-21T10:45:16.702 INFO:teuthology.orchestra.run.smithi123.stdout: "in_progress": true, 2024-09-21T10:45:16.702 INFO:teuthology.orchestra.run.smithi123.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T10:45:16.702 INFO:teuthology.orchestra.run.smithi123.stdout: "services_complete": [ 2024-09-21T10:45:16.702 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr", 2024-09-21T10:45:16.702 INFO:teuthology.orchestra.run.smithi123.stdout: "crash", 2024-09-21T10:45:16.702 INFO:teuthology.orchestra.run.smithi123.stdout: "mon" 2024-09-21T10:45:16.702 INFO:teuthology.orchestra.run.smithi123.stdout: ], 2024-09-21T10:45:16.702 INFO:teuthology.orchestra.run.smithi123.stdout: "progress": "8/23 daemons upgraded", 2024-09-21T10:45:16.702 INFO:teuthology.orchestra.run.smithi123.stdout: "message": "Currently upgrading osd daemons", 2024-09-21T10:45:16.703 INFO:teuthology.orchestra.run.smithi123.stdout: "is_paused": false 2024-09-21T10:45:16.703 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-09-21T10:45:16.836 INFO:teuthology.orchestra.run.smithi123.stdout: 5 205323 10.36 MB/sec execute 377 sec latency 51.188 ms 2024-09-21T10:45:17.255 INFO:teuthology.orchestra.run.smithi123.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 151/1668 objects degraded (9.053%), 20 pgs degraded 2024-09-21T10:45:17.256 INFO:teuthology.orchestra.run.smithi123.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-21T10:45:17.256 INFO:teuthology.orchestra.run.smithi123.stdout: osd.2 (root=default,host=smithi123) is down 2024-09-21T10:45:17.256 INFO:teuthology.orchestra.run.smithi123.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 151/1668 objects degraded (9.053%), 20 pgs degraded 2024-09-21T10:45:17.256 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.0 is active+undersized+degraded, acting [1,0] 2024-09-21T10:45:17.256 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.1 is active+undersized+degraded, acting [3,0] 2024-09-21T10:45:17.256 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.3 is active+undersized+degraded, acting [5,7] 2024-09-21T10:45:17.256 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.7 is active+undersized+degraded, acting [6,7] 2024-09-21T10:45:17.256 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.c is active+undersized+degraded, acting [5,0] 2024-09-21T10:45:17.256 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.e is active+undersized+degraded, acting [3,7] 2024-09-21T10:45:17.256 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.10 is active+undersized+degraded, acting [1,0] 2024-09-21T10:45:17.257 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.16 is active+undersized+degraded, acting [5,6] 2024-09-21T10:45:17.257 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.17 is active+undersized+degraded, acting [6,5] 2024-09-21T10:45:17.257 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.1c is active+undersized+degraded, acting [7,5] 2024-09-21T10:45:17.257 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.0 is active+undersized+degraded, acting [1,6] 2024-09-21T10:45:17.257 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.4 is active+undersized+degraded, acting [1,5] 2024-09-21T10:45:17.257 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.5 is active+undersized+degraded, acting [5,3] 2024-09-21T10:45:17.257 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.9 is active+undersized+degraded, acting [4,7] 2024-09-21T10:45:17.257 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.13 is active+undersized+degraded, acting [7,4] 2024-09-21T10:45:17.257 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.1a is active+undersized+degraded, acting [4,1] 2024-09-21T10:45:17.257 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.1e is active+undersized+degraded, acting [3,6] 2024-09-21T10:45:17.257 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.1f is active+undersized+degraded, acting [0,5] 2024-09-21T10:45:17.257 INFO:teuthology.orchestra.run.smithi123.stdout: pg 4.14 is active+undersized+degraded, acting [3,7] 2024-09-21T10:45:17.257 INFO:teuthology.orchestra.run.smithi123.stdout: pg 4.19 is active+undersized+degraded, acting [3,0] 2024-09-21T10:45:17.470 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:17 smithi123 ceph-mon[63856]: pgmap v144: 97 pgs: 7 active+undersized, 20 active+undersized+degraded, 70 active+clean; 347 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 7.2 MiB/s wr, 682 op/s; 151/1668 objects degraded (9.053%) 2024-09-21T10:45:17.470 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:17 smithi123 ceph-mon[63856]: from='client.34294 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:45:17.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:17 smithi175 ceph-mon[49304]: pgmap v144: 97 pgs: 7 active+undersized, 20 active+undersized+degraded, 70 active+clean; 347 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 7.2 MiB/s wr, 682 op/s; 151/1668 objects degraded (9.053%) 2024-09-21T10:45:17.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:17 smithi175 ceph-mon[49304]: from='client.34294 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:45:17.836 INFO:teuthology.orchestra.run.smithi123.stdout: 5 205513 10.34 MB/sec execute 378 sec latency 61.839 ms 2024-09-21T10:45:18.265 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:18 smithi123 ceph-mon[63856]: from='client.? 172.21.15.123:0/1750190173' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T10:45:18.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:18 smithi175 ceph-mon[49304]: from='client.? 172.21.15.123:0/1750190173' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T10:45:18.836 INFO:teuthology.orchestra.run.smithi123.stdout: 5 205690 10.33 MB/sec execute 379 sec latency 124.778 ms 2024-09-21T10:45:19.285 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:19 smithi123 ceph-mon[63856]: pgmap v145: 97 pgs: 7 active+undersized, 20 active+undersized+degraded, 70 active+clean; 344 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 7.2 MiB/s wr, 689 op/s; 157/1659 objects degraded (9.464%) 2024-09-21T10:45:19.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:19 smithi175 ceph-mon[49304]: pgmap v145: 97 pgs: 7 active+undersized, 20 active+undersized+degraded, 70 active+clean; 344 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 7.2 MiB/s wr, 689 op/s; 157/1659 objects degraded (9.464%) 2024-09-21T10:45:19.836 INFO:teuthology.orchestra.run.smithi123.stdout: 5 205922 10.32 MB/sec execute 380 sec latency 72.666 ms 2024-09-21T10:45:20.299 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:20 smithi123 ceph-mon[63856]: Health check update: Degraded data redundancy: 157/1659 objects degraded (9.464%), 20 pgs degraded (PG_DEGRADED) 2024-09-21T10:45:20.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:20 smithi175 ceph-mon[49304]: Health check update: Degraded data redundancy: 157/1659 objects degraded (9.464%), 20 pgs degraded (PG_DEGRADED) 2024-09-21T10:45:20.836 INFO:teuthology.orchestra.run.smithi123.stdout: 5 206169 10.31 MB/sec execute 381 sec latency 131.950 ms 2024-09-21T10:45:21.317 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:21 smithi123 ceph-mon[63856]: pgmap v146: 97 pgs: 7 active+undersized, 20 active+undersized+degraded, 70 active+clean; 348 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.8 MiB/s wr, 877 op/s; 156/1641 objects degraded (9.506%) 2024-09-21T10:45:21.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:21 smithi175 ceph-mon[49304]: pgmap v146: 97 pgs: 7 active+undersized, 20 active+undersized+degraded, 70 active+clean; 348 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.8 MiB/s wr, 877 op/s; 156/1641 objects degraded (9.506%) 2024-09-21T10:45:21.836 INFO:teuthology.orchestra.run.smithi123.stdout: 5 206433 10.30 MB/sec execute 382 sec latency 77.922 ms 2024-09-21T10:45:22.081 INFO:journalctl@ceph.osd.2.smithi123.stdout:Sep 21 10:45:21 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-2[73624]: 2024-09-21T10:45:21.834+0000 7fb594ec0640 -1 osd.2 0 read_superblock omap replica is missing. 2024-09-21T10:45:22.836 INFO:teuthology.orchestra.run.smithi123.stdout: 5 206661 10.29 MB/sec execute 383 sec latency 56.158 ms 2024-09-21T10:45:23.127 INFO:journalctl@ceph.osd.2.smithi123.stdout:Sep 21 10:45:22 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-2[73624]: 2024-09-21T10:45:22.891+0000 7fb594ec0640 -1 osd.2 108 log_to_monitors true 2024-09-21T10:45:23.381 INFO:journalctl@ceph.osd.2.smithi123.stdout:Sep 21 10:45:23 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-2[73624]: 2024-09-21T10:45:23.348+0000 7fb58cc6a640 -1 osd.2 108 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T10:45:23.381 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:23 smithi123 ceph-mon[63856]: pgmap v147: 97 pgs: 7 active+undersized, 20 active+undersized+degraded, 70 active+clean; 348 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.5 MiB/s wr, 539 op/s; 156/1641 objects degraded (9.506%) 2024-09-21T10:45:23.382 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:23 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:45:23.382 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:23 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:45:23.382 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:23 smithi123 ceph-mon[63856]: from='osd.2 [v2:172.21.15.123:6818/1612375293,v1:172.21.15.123:6819/1612375293]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-21T10:45:23.382 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:23 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:45:23.382 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:23 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:45:23.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:23 smithi175 ceph-mon[49304]: pgmap v147: 97 pgs: 7 active+undersized, 20 active+undersized+degraded, 70 active+clean; 348 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.5 MiB/s wr, 539 op/s; 156/1641 objects degraded (9.506%) 2024-09-21T10:45:23.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:23 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:45:23.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:23 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:45:23.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:23 smithi175 ceph-mon[49304]: from='osd.2 [v2:172.21.15.123:6818/1612375293,v1:172.21.15.123:6819/1612375293]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-21T10:45:23.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:23 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:45:23.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:23 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:45:23.837 INFO:teuthology.orchestra.run.smithi123.stdout: 5 206853 10.28 MB/sec execute 384 sec latency 53.233 ms 2024-09-21T10:45:24.403 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:24 smithi123 ceph-mon[63856]: from='osd.2 [v2:172.21.15.123:6818/1612375293,v1:172.21.15.123:6819/1612375293]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-21T10:45:24.403 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:24 smithi123 ceph-mon[63856]: osdmap e111: 8 total, 7 up, 8 in 2024-09-21T10:45:24.403 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:24 smithi123 ceph-mon[63856]: from='osd.2 [v2:172.21.15.123:6818/1612375293,v1:172.21.15.123:6819/1612375293]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi123", "root=default"]} : dispatch 2024-09-21T10:45:24.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:24 smithi175 ceph-mon[49304]: from='osd.2 [v2:172.21.15.123:6818/1612375293,v1:172.21.15.123:6819/1612375293]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-21T10:45:24.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:24 smithi175 ceph-mon[49304]: osdmap e111: 8 total, 7 up, 8 in 2024-09-21T10:45:24.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:24 smithi175 ceph-mon[49304]: from='osd.2 [v2:172.21.15.123:6818/1612375293,v1:172.21.15.123:6819/1612375293]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi123", "root=default"]} : dispatch 2024-09-21T10:45:24.837 INFO:teuthology.orchestra.run.smithi123.stdout: 5 207091 10.27 MB/sec execute 385 sec latency 504.877 ms 2024-09-21T10:45:25.480 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:25 smithi123 ceph-mon[63856]: pgmap v149: 97 pgs: 7 active+undersized, 20 active+undersized+degraded, 70 active+clean; 345 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.1 MiB/s wr, 465 op/s; 146/1527 objects degraded (9.561%) 2024-09-21T10:45:25.480 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:25 smithi123 ceph-mon[63856]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T10:45:25.480 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:25 smithi123 ceph-mon[63856]: osd.2 [v2:172.21.15.123:6818/1612375293,v1:172.21.15.123:6819/1612375293] boot 2024-09-21T10:45:25.480 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:25 smithi123 ceph-mon[63856]: osdmap e112: 8 total, 8 up, 8 in 2024-09-21T10:45:25.480 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:25 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-21T10:45:25.480 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:25 smithi123 ceph-mon[63856]: Health check update: Degraded data redundancy: 146/1527 objects degraded (9.561%), 20 pgs degraded (PG_DEGRADED) 2024-09-21T10:45:25.480 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:25 smithi123 ceph-mon[63856]: osdmap e113: 8 total, 8 up, 8 in 2024-09-21T10:45:25.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:25 smithi175 ceph-mon[49304]: pgmap v149: 97 pgs: 7 active+undersized, 20 active+undersized+degraded, 70 active+clean; 345 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.1 MiB/s wr, 465 op/s; 146/1527 objects degraded (9.561%) 2024-09-21T10:45:25.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:25 smithi175 ceph-mon[49304]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T10:45:25.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:25 smithi175 ceph-mon[49304]: osd.2 [v2:172.21.15.123:6818/1612375293,v1:172.21.15.123:6819/1612375293] boot 2024-09-21T10:45:25.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:25 smithi175 ceph-mon[49304]: osdmap e112: 8 total, 8 up, 8 in 2024-09-21T10:45:25.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:25 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-21T10:45:25.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:25 smithi175 ceph-mon[49304]: Health check update: Degraded data redundancy: 146/1527 objects degraded (9.561%), 20 pgs degraded (PG_DEGRADED) 2024-09-21T10:45:25.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:25 smithi175 ceph-mon[49304]: osdmap e113: 8 total, 8 up, 8 in 2024-09-21T10:45:25.837 INFO:teuthology.orchestra.run.smithi123.stdout: 5 207163 10.24 MB/sec execute 386 sec latency 1504.922 ms 2024-09-21T10:45:26.837 INFO:teuthology.orchestra.run.smithi123.stdout: 5 207405 10.23 MB/sec execute 387 sec latency 1832.423 ms 2024-09-21T10:45:27.182 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:27 smithi123 ceph-mon[63856]: osdmap e114: 8 total, 8 up, 8 in 2024-09-21T10:45:27.182 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:27 smithi123 ceph-mon[63856]: pgmap v153: 97 pgs: 2 peering, 6 active+undersized, 19 active+undersized+degraded, 70 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 4.4 MiB/s wr, 432 op/s; 138/1545 objects degraded (8.932%) 2024-09-21T10:45:27.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:27 smithi175 ceph-mon[49304]: osdmap e114: 8 total, 8 up, 8 in 2024-09-21T10:45:27.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:27 smithi175 ceph-mon[49304]: pgmap v153: 97 pgs: 2 peering, 6 active+undersized, 19 active+undersized+degraded, 70 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 4.4 MiB/s wr, 432 op/s; 138/1545 objects degraded (8.932%) 2024-09-21T10:45:27.837 INFO:teuthology.orchestra.run.smithi123.stdout: 5 207724 10.21 MB/sec execute 388 sec latency 82.111 ms 2024-09-21T10:45:28.235 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:28 smithi123 ceph-mon[63856]: osdmap e115: 8 total, 8 up, 8 in 2024-09-21T10:45:28.235 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:28 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:45:28.235 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:28 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:45:28.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:28 smithi175 ceph-mon[49304]: osdmap e115: 8 total, 8 up, 8 in 2024-09-21T10:45:28.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:28 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:45:28.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:28 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:45:28.837 INFO:teuthology.orchestra.run.smithi123.stdout: 5 207885 10.21 MB/sec execute 389 sec latency 645.436 ms 2024-09-21T10:45:29.266 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:29 smithi123 ceph-mon[63856]: osdmap e116: 8 total, 8 up, 8 in 2024-09-21T10:45:29.266 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:29 smithi123 ceph-mon[63856]: pgmap v156: 97 pgs: 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 peering, 2 active+undersized, 15 active+undersized+degraded, 74 active+clean; 342 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 1.9 MiB/s wr, 188 op/s; 139/1539 objects degraded (9.032%); 128 KiB/s, 17 objects/s recovering 2024-09-21T10:45:29.266 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:29 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:45:29.266 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:29 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:45:29.266 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:29 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:45:29.266 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:29 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:45:29.267 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:29 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:45:29.267 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:29 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:45:29.267 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:29 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:45:29.267 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:29 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:45:29.267 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:29 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:45:29.267 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:29 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:45:29.267 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:29 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-21T10:45:29.267 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:29 smithi123 ceph-mon[63856]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-21T10:45:29.267 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:29 smithi123 ceph-mon[63856]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-09-21T10:45:29.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:29 smithi175 ceph-mon[49304]: osdmap e116: 8 total, 8 up, 8 in 2024-09-21T10:45:29.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:29 smithi175 ceph-mon[49304]: pgmap v156: 97 pgs: 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 peering, 2 active+undersized, 15 active+undersized+degraded, 74 active+clean; 342 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 1.9 MiB/s wr, 188 op/s; 139/1539 objects degraded (9.032%); 128 KiB/s, 17 objects/s recovering 2024-09-21T10:45:29.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:29 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:45:29.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:29 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:45:29.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:29 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:45:29.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:29 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:45:29.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:29 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:45:29.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:29 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:45:29.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:29 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:45:29.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:29 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:45:29.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:29 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:45:29.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:29 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:45:29.375 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:29 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-21T10:45:29.375 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:29 smithi175 ceph-mon[49304]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-21T10:45:29.375 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:29 smithi175 ceph-mon[49304]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-09-21T10:45:29.837 INFO:teuthology.orchestra.run.smithi123.stdout: 5 208163 10.21 MB/sec execute 390 sec latency 913.146 ms 2024-09-21T10:45:30.288 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:30 smithi123 ceph-mon[63856]: osdmap e117: 8 total, 8 up, 8 in 2024-09-21T10:45:30.288 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:30 smithi123 ceph-mon[63856]: Health check update: Degraded data redundancy: 139/1539 objects degraded (9.032%), 17 pgs degraded (PG_DEGRADED) 2024-09-21T10:45:30.288 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:30 smithi123 ceph-mon[63856]: osdmap e118: 8 total, 8 up, 8 in 2024-09-21T10:45:30.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:30 smithi175 ceph-mon[49304]: osdmap e117: 8 total, 8 up, 8 in 2024-09-21T10:45:30.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:30 smithi175 ceph-mon[49304]: Health check update: Degraded data redundancy: 139/1539 objects degraded (9.032%), 17 pgs degraded (PG_DEGRADED) 2024-09-21T10:45:30.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:30 smithi175 ceph-mon[49304]: osdmap e118: 8 total, 8 up, 8 in 2024-09-21T10:45:30.837 INFO:teuthology.orchestra.run.smithi123.stdout: 5 208443 10.20 MB/sec execute 391 sec latency 755.845 ms 2024-09-21T10:45:31.291 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:31 smithi123 ceph-mon[63856]: pgmap v159: 97 pgs: 1 active+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 peering, 1 active+recovering+undersized+degraded+remapped, 90 active+clean; 342 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.6 MiB/s wr, 413 op/s; 107/1509 objects degraded (7.091%); 2.2 MiB/s, 28 objects/s recovering 2024-09-21T10:45:31.291 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:31 smithi123 ceph-mon[63856]: osdmap e119: 8 total, 8 up, 8 in 2024-09-21T10:45:31.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:31 smithi175 ceph-mon[49304]: pgmap v159: 97 pgs: 1 active+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 peering, 1 active+recovering+undersized+degraded+remapped, 90 active+clean; 342 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.6 MiB/s wr, 413 op/s; 107/1509 objects degraded (7.091%); 2.2 MiB/s, 28 objects/s recovering 2024-09-21T10:45:31.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:31 smithi175 ceph-mon[49304]: osdmap e119: 8 total, 8 up, 8 in 2024-09-21T10:45:31.837 INFO:teuthology.orchestra.run.smithi123.stdout: 5 208615 10.18 MB/sec execute 392 sec latency 958.113 ms 2024-09-21T10:45:32.837 INFO:teuthology.orchestra.run.smithi123.stdout: 5 208941 10.19 MB/sec execute 393 sec latency 884.918 ms 2024-09-21T10:45:33.086 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:33 smithi123 ceph-mon[63856]: osdmap e120: 8 total, 8 up, 8 in 2024-09-21T10:45:33.086 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:33 smithi123 ceph-mon[63856]: pgmap v162: 97 pgs: 1 active+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 peering, 1 active+recovering+undersized+degraded+remapped, 90 active+clean; 342 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 2.2 MiB/s wr, 253 op/s; 107/1509 objects degraded (7.091%); 2.0 MiB/s, 11 objects/s recovering 2024-09-21T10:45:33.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:33 smithi175 ceph-mon[49304]: osdmap e120: 8 total, 8 up, 8 in 2024-09-21T10:45:33.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:33 smithi175 ceph-mon[49304]: pgmap v162: 97 pgs: 1 active+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 peering, 1 active+recovering+undersized+degraded+remapped, 90 active+clean; 342 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 2.2 MiB/s wr, 253 op/s; 107/1509 objects degraded (7.091%); 2.0 MiB/s, 11 objects/s recovering 2024-09-21T10:45:33.837 INFO:teuthology.orchestra.run.smithi123.stdout: 5 209232 10.19 MB/sec execute 394 sec latency 41.912 ms 2024-09-21T10:45:34.370 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:34 smithi123 ceph-mon[63856]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 107/1509 objects degraded (7.091%), 5 pgs degraded) 2024-09-21T10:45:34.370 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:34 smithi123 ceph-mon[63856]: Cluster is now healthy 2024-09-21T10:45:34.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:34 smithi175 ceph-mon[49304]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 107/1509 objects degraded (7.091%), 5 pgs degraded) 2024-09-21T10:45:34.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:34 smithi175 ceph-mon[49304]: Cluster is now healthy 2024-09-21T10:45:34.837 INFO:teuthology.orchestra.run.smithi123.stdout: 5 209495 10.19 MB/sec execute 395 sec latency 64.712 ms 2024-09-21T10:45:35.404 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:35 smithi123 ceph-mon[63856]: pgmap v163: 97 pgs: 97 active+clean; 347 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3.9 KiB/s rd, 3.5 MiB/s wr, 340 op/s; 59 KiB/s, 28 objects/s recovering 2024-09-21T10:45:35.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:35 smithi175 ceph-mon[49304]: pgmap v163: 97 pgs: 97 active+clean; 347 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3.9 KiB/s rd, 3.5 MiB/s wr, 340 op/s; 59 KiB/s, 28 objects/s recovering 2024-09-21T10:45:35.837 INFO:teuthology.orchestra.run.smithi123.stdout: 5 209736 10.17 MB/sec execute 396 sec latency 53.306 ms 2024-09-21T10:45:36.837 INFO:teuthology.orchestra.run.smithi123.stdout: 5 209949 10.16 MB/sec execute 397 sec latency 99.671 ms 2024-09-21T10:45:37.502 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:37 smithi123 ceph-mon[63856]: pgmap v164: 97 pgs: 97 active+clean; 349 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3.2 KiB/s rd, 3.1 MiB/s wr, 280 op/s; 46 KiB/s, 19 objects/s recovering 2024-09-21T10:45:37.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:37 smithi175 ceph-mon[49304]: pgmap v164: 97 pgs: 97 active+clean; 349 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3.2 KiB/s rd, 3.1 MiB/s wr, 280 op/s; 46 KiB/s, 19 objects/s recovering 2024-09-21T10:45:37.837 INFO:teuthology.orchestra.run.smithi123.stdout: 5 210199 10.15 MB/sec execute 398 sec latency 63.913 ms 2024-09-21T10:45:38.837 INFO:teuthology.orchestra.run.smithi123.stdout: 5 210444 10.13 MB/sec execute 399 sec latency 57.575 ms 2024-09-21T10:45:39.543 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:39 smithi123 ceph-mon[63856]: pgmap v165: 97 pgs: 97 active+clean; 349 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 2.5 KiB/s rd, 2.9 MiB/s wr, 240 op/s; 36 KiB/s, 15 objects/s recovering 2024-09-21T10:45:39.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:39 smithi175 ceph-mon[49304]: pgmap v165: 97 pgs: 97 active+clean; 349 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 2.5 KiB/s rd, 2.9 MiB/s wr, 240 op/s; 36 KiB/s, 15 objects/s recovering 2024-09-21T10:45:39.837 INFO:teuthology.orchestra.run.smithi123.stdout: 5 210711 10.12 MB/sec execute 400 sec latency 66.173 ms 2024-09-21T10:45:40.837 INFO:teuthology.orchestra.run.smithi123.stdout: 5 211079 10.11 MB/sec execute 401 sec latency 38.225 ms 2024-09-21T10:45:41.591 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:41 smithi123 ceph-mon[63856]: pgmap v166: 97 pgs: 97 active+clean; 346 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.3 MiB/s wr, 409 op/s; 31 KiB/s, 13 objects/s recovering 2024-09-21T10:45:41.837 INFO:teuthology.orchestra.run.smithi123.stdout: 5 211474 10.11 MB/sec execute 402 sec latency 50.078 ms 2024-09-21T10:45:41.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:41 smithi175 ceph-mon[49304]: pgmap v166: 97 pgs: 97 active+clean; 346 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.3 MiB/s wr, 409 op/s; 31 KiB/s, 13 objects/s recovering 2024-09-21T10:45:42.837 INFO:teuthology.orchestra.run.smithi123.stdout: 5 212002 10.14 MB/sec execute 403 sec latency 35.032 ms 2024-09-21T10:45:43.837 INFO:teuthology.orchestra.run.smithi123.stdout: 5 212579 10.16 MB/sec execute 404 sec latency 34.488 ms 2024-09-21T10:45:43.859 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:43 smithi123 ceph-mon[63856]: pgmap v167: 97 pgs: 97 active+clean; 346 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.9 MiB/s wr, 369 op/s; 28 KiB/s, 12 objects/s recovering 2024-09-21T10:45:43.859 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:43 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:45:43.859 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:43 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:45:43.860 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:43 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:45:43.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:43 smithi175 ceph-mon[49304]: pgmap v167: 97 pgs: 97 active+clean; 346 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.9 MiB/s wr, 369 op/s; 28 KiB/s, 12 objects/s recovering 2024-09-21T10:45:43.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:43 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:45:43.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:43 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:45:43.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:43 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:45:44.634 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:44 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-21T10:45:44.634 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:44 smithi123 ceph-mon[63856]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-21T10:45:44.634 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:44 smithi123 ceph-mon[63856]: Upgrade: osd.3 is safe to restart 2024-09-21T10:45:44.838 INFO:teuthology.orchestra.run.smithi123.stdout: 5 212985 10.16 MB/sec execute 405 sec latency 40.914 ms 2024-09-21T10:45:45.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:44 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-21T10:45:45.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:44 smithi175 ceph-mon[49304]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-21T10:45:45.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:44 smithi175 ceph-mon[49304]: Upgrade: osd.3 is safe to restart 2024-09-21T10:45:45.803 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:45 smithi123 ceph-mon[63856]: pgmap v168: 97 pgs: 97 active+clean; 348 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.9 MiB/s wr, 512 op/s; 24 KiB/s, 10 objects/s recovering 2024-09-21T10:45:45.837 INFO:teuthology.orchestra.run.smithi123.stdout: 5 213339 10.16 MB/sec execute 406 sec latency 49.384 ms 2024-09-21T10:45:46.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:45 smithi175 ceph-mon[49304]: pgmap v168: 97 pgs: 97 active+clean; 348 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.9 MiB/s wr, 512 op/s; 24 KiB/s, 10 objects/s recovering 2024-09-21T10:45:46.838 INFO:teuthology.orchestra.run.smithi123.stdout: 5 213686 10.15 MB/sec execute 407 sec latency 40.870 ms 2024-09-21T10:45:47.569 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:47 smithi123 ceph-mon[63856]: pgmap v169: 97 pgs: 97 active+clean; 351 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.1 MiB/s wr, 396 op/s 2024-09-21T10:45:47.569 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:47 smithi123 ceph-mon[63856]: Upgrade: Updating osd.3 2024-09-21T10:45:47.569 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:47 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:45:47.569 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:47 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-21T10:45:47.569 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:47 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:45:47.570 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:47 smithi123 ceph-mon[63856]: Deploying daemon osd.3 on smithi123 2024-09-21T10:45:47.818 INFO:teuthology.orchestra.run.smithi123.stdout:true 2024-09-21T10:45:47.838 INFO:teuthology.orchestra.run.smithi123.stdout: 5 213974 10.14 MB/sec execute 408 sec latency 37.604 ms 2024-09-21T10:45:47.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:47 smithi175 ceph-mon[49304]: pgmap v169: 97 pgs: 97 active+clean; 351 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.1 MiB/s wr, 396 op/s 2024-09-21T10:45:47.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:47 smithi175 ceph-mon[49304]: Upgrade: Updating osd.3 2024-09-21T10:45:47.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:47 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:45:47.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:47 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-21T10:45:47.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:47 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:45:47.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:47 smithi175 ceph-mon[49304]: Deploying daemon osd.3 on smithi123 2024-09-21T10:45:48.600 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:48 smithi123 ceph-mon[63856]: from='client.34304 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:45:48.783 INFO:teuthology.orchestra.run.smithi123.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T10:45:48.783 INFO:teuthology.orchestra.run.smithi123.stdout:alertmanager.smithi123 smithi123 *:9093,9094 running (6m) 26s ago 15m 22.0M - 0.27.0 11f11916f8cd 5d550bd97de5 2024-09-21T10:45:48.783 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi123 smithi123 running (3m) 26s ago 15m 6899k - 19.3.0-5103-g5c1cdc3c 8f4d013db793 c5e1ef39a0a8 2024-09-21T10:45:48.783 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi175 smithi175 running (2m) 2m ago 14m 6895k - 19.3.0-5103-g5c1cdc3c 8f4d013db793 fb1ca999ccfd 2024-09-21T10:45:48.783 INFO:teuthology.orchestra.run.smithi123.stdout:grafana.smithi123 smithi123 *:3000 running (6m) 26s ago 15m 50.0M - dad864ee21e9 771df81763ba 2024-09-21T10:45:48.783 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi123.uqoyow smithi123 running (9m) 26s ago 9m 40.9M - 17.2.0 e1d6a67b021e f0a21eaac266 2024-09-21T10:45:48.784 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi175.jrlevw smithi175 running (9m) 2m ago 9m 13.5M - 17.2.0 e1d6a67b021e b21b16f4bc50 2024-09-21T10:45:48.784 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi123.stnuit smithi123 *:8443,9283,8765 running (5m) 26s ago 16m 518M - 19.3.0-5103-g5c1cdc3c 8f4d013db793 6812cc5cb56b 2024-09-21T10:45:48.784 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi175.txvwrq smithi175 *:8443,9283,8765 running (4m) 2m ago 14m 453M - 19.3.0-5103-g5c1cdc3c 8f4d013db793 304f0408075e 2024-09-21T10:45:48.784 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi123 smithi123 running (4m) 26s ago 16m 50.6M 2048M 19.3.0-5103-g5c1cdc3c 8f4d013db793 6b95207a2d80 2024-09-21T10:45:48.784 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi175 smithi175 running (3m) 2m ago 14m 36.7M 2048M 19.3.0-5103-g5c1cdc3c 8f4d013db793 3abee2d37f0f 2024-09-21T10:45:48.784 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.0.0.smithi123.ddatpr smithi123 *:2049 running (9m) 26s ago 9m 279M - 4.0 e1d6a67b021e 36f8cb2281eb 2024-09-21T10:45:48.784 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.1.0.smithi175.wpawer smithi175 *:2049 running (8m) 2m ago 8m 90.3M - 4.0 e1d6a67b021e 2200aeed9200 2024-09-21T10:45:48.784 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi123 smithi123 *:9100 running (5m) 26s ago 15m 9.96M - 1.7.0 72c9c2088986 ca7d0b29af81 2024-09-21T10:45:48.784 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi175 smithi175 *:9100 running (5m) 2m ago 14m 11.1M - 1.7.0 72c9c2088986 46c4f6b7c149 2024-09-21T10:45:48.784 INFO:teuthology.orchestra.run.smithi123.stdout:osd.0 smithi123 running (2m) 26s ago 13m 309M 1936M 19.3.0-5103-g5c1cdc3c 8f4d013db793 cc40f695c3a8 2024-09-21T10:45:48.784 INFO:teuthology.orchestra.run.smithi123.stdout:osd.1 smithi123 running (86s) 26s ago 13m 238M 1936M 19.3.0-5103-g5c1cdc3c 8f4d013db793 1be249025c7f 2024-09-21T10:45:48.785 INFO:teuthology.orchestra.run.smithi123.stdout:osd.2 smithi123 running (35s) 26s ago 12m 12.2M 1936M 19.3.0-5103-g5c1cdc3c 8f4d013db793 647aec759e3c 2024-09-21T10:45:48.785 INFO:teuthology.orchestra.run.smithi123.stdout:osd.3 smithi123 running (12m) 26s ago 12m 374M 1936M 17.2.0 e1d6a67b021e c1672dea7b1f 2024-09-21T10:45:48.785 INFO:teuthology.orchestra.run.smithi123.stdout:osd.4 smithi175 running (12m) 2m ago 12m 498M 2704M 17.2.0 e1d6a67b021e 48fe728c259e 2024-09-21T10:45:48.785 INFO:teuthology.orchestra.run.smithi123.stdout:osd.5 smithi175 running (11m) 2m ago 11m 331M 2704M 17.2.0 e1d6a67b021e e270d5a21b14 2024-09-21T10:45:48.785 INFO:teuthology.orchestra.run.smithi123.stdout:osd.6 smithi175 running (11m) 2m ago 11m 517M 2704M 17.2.0 e1d6a67b021e 9bcbf018eb20 2024-09-21T10:45:48.785 INFO:teuthology.orchestra.run.smithi123.stdout:osd.7 smithi175 running (10m) 2m ago 10m 383M 2704M 17.2.0 e1d6a67b021e e9b8f1f90994 2024-09-21T10:45:48.785 INFO:teuthology.orchestra.run.smithi123.stdout:prometheus.smithi123 smithi123 *:9095 running (4m) 26s ago 14m 53.1M - 2.51.0 1d3b7f56885b 37f944c9de88 2024-09-21T10:45:48.838 INFO:teuthology.orchestra.run.smithi123.stdout: 5 214277 10.13 MB/sec execute 409 sec latency 96.811 ms 2024-09-21T10:45:48.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:48 smithi175 ceph-mon[49304]: from='client.34304 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:45:49.380 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-09-21T10:45:49.381 INFO:teuthology.orchestra.run.smithi123.stdout: "mon": { 2024-09-21T10:45:49.381 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev)": 2 2024-09-21T10:45:49.381 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:45:49.381 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr": { 2024-09-21T10:45:49.381 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev)": 2 2024-09-21T10:45:49.381 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:45:49.381 INFO:teuthology.orchestra.run.smithi123.stdout: "osd": { 2024-09-21T10:45:49.381 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-09-21T10:45:49.381 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev)": 3 2024-09-21T10:45:49.381 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:45:49.381 INFO:teuthology.orchestra.run.smithi123.stdout: "mds": { 2024-09-21T10:45:49.381 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T10:45:49.382 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:45:49.382 INFO:teuthology.orchestra.run.smithi123.stdout: "overall": { 2024-09-21T10:45:49.382 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-09-21T10:45:49.382 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev)": 7 2024-09-21T10:45:49.382 INFO:teuthology.orchestra.run.smithi123.stdout: } 2024-09-21T10:45:49.382 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-09-21T10:45:49.643 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:49 smithi123 ceph-mon[63856]: pgmap v170: 97 pgs: 97 active+clean; 354 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.4 MiB/s wr, 470 op/s 2024-09-21T10:45:49.643 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:49 smithi123 ceph-mon[63856]: from='client.34308 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:45:49.643 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:49 smithi123 ceph-mon[63856]: from='client.34312 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:45:49.643 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:49 smithi123 ceph-mon[63856]: from='client.? 172.21.15.123:0/2522374898' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:45:49.829 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-09-21T10:45:49.829 INFO:teuthology.orchestra.run.smithi123.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472", 2024-09-21T10:45:49.829 INFO:teuthology.orchestra.run.smithi123.stdout: "in_progress": true, 2024-09-21T10:45:49.829 INFO:teuthology.orchestra.run.smithi123.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T10:45:49.829 INFO:teuthology.orchestra.run.smithi123.stdout: "services_complete": [ 2024-09-21T10:45:49.829 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr", 2024-09-21T10:45:49.829 INFO:teuthology.orchestra.run.smithi123.stdout: "crash", 2024-09-21T10:45:49.830 INFO:teuthology.orchestra.run.smithi123.stdout: "mon" 2024-09-21T10:45:49.830 INFO:teuthology.orchestra.run.smithi123.stdout: ], 2024-09-21T10:45:49.830 INFO:teuthology.orchestra.run.smithi123.stdout: "progress": "9/23 daemons upgraded", 2024-09-21T10:45:49.830 INFO:teuthology.orchestra.run.smithi123.stdout: "message": "Currently upgrading osd daemons", 2024-09-21T10:45:49.830 INFO:teuthology.orchestra.run.smithi123.stdout: "is_paused": false 2024-09-21T10:45:49.830 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-09-21T10:45:49.838 INFO:teuthology.orchestra.run.smithi123.stdout: 5 214569 10.11 MB/sec execute 410 sec latency 73.529 ms 2024-09-21T10:45:49.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:49 smithi175 ceph-mon[49304]: pgmap v170: 97 pgs: 97 active+clean; 354 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.4 MiB/s wr, 470 op/s 2024-09-21T10:45:49.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:49 smithi175 ceph-mon[49304]: from='client.34308 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:45:49.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:49 smithi175 ceph-mon[49304]: from='client.34312 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:45:49.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:49 smithi175 ceph-mon[49304]: from='client.? 172.21.15.123:0/2522374898' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:45:50.454 INFO:teuthology.orchestra.run.smithi123.stdout:HEALTH_OK 2024-09-21T10:45:50.654 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:50 smithi123 ceph-mon[63856]: from='client.? 172.21.15.123:0/371409910' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T10:45:50.838 INFO:teuthology.orchestra.run.smithi123.stdout: 5 214898 10.12 MB/sec execute 411 sec latency 52.956 ms 2024-09-21T10:45:50.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:50 smithi175 ceph-mon[49304]: from='client.? 172.21.15.123:0/371409910' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T10:45:50.905 INFO:journalctl@ceph.osd.3.smithi123.stdout:Sep 21 10:45:50 smithi123 systemd[1]: Stopping Ceph osd.3 for 3ccd15e6-7804-11ef-baf6-efdc52797490... 2024-09-21T10:45:51.247 INFO:journalctl@ceph.osd.3.smithi123.stdout:Sep 21 10:45:51 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-3[45202]: 2024-09-21T10:45:51.194+0000 7efd00cfc700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:45:51.247 INFO:journalctl@ceph.osd.3.smithi123.stdout:Sep 21 10:45:51 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-3[45202]: 2024-09-21T10:45:51.194+0000 7efd00cfc700 -1 osd.3 120 *** Got signal Terminated *** 2024-09-21T10:45:51.247 INFO:journalctl@ceph.osd.3.smithi123.stdout:Sep 21 10:45:51 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-3[45202]: 2024-09-21T10:45:51.194+0000 7efd00cfc700 -1 osd.3 120 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T10:45:51.661 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:51 smithi123 ceph-mon[63856]: from='client.34320 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:45:51.661 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:51 smithi123 ceph-mon[63856]: pgmap v171: 97 pgs: 97 active+clean; 351 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.9 KiB/s rd, 5.9 MiB/s wr, 616 op/s 2024-09-21T10:45:51.661 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:51 smithi123 ceph-mon[63856]: osd.3 marked itself down and dead 2024-09-21T10:45:51.838 INFO:teuthology.orchestra.run.smithi123.stdout: 5 215228 10.10 MB/sec execute 412 sec latency 250.802 ms 2024-09-21T10:45:51.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:51 smithi175 ceph-mon[49304]: from='client.34320 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:45:51.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:51 smithi175 ceph-mon[49304]: pgmap v171: 97 pgs: 97 active+clean; 351 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.9 KiB/s rd, 5.9 MiB/s wr, 616 op/s 2024-09-21T10:45:51.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:51 smithi175 ceph-mon[49304]: osd.3 marked itself down and dead 2024-09-21T10:45:52.508 INFO:journalctl@ceph.osd.3.smithi123.stdout:Sep 21 10:45:52 smithi123 podman[75643]: 2024-09-21 10:45:52.141049925 +0000 UTC m=+1.087060090 container died c1672dea7b1fd4a87284f6559bb91234b1dc580559cfeeced7f8179922027e8d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-3, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, name=centos-stream, io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, ceph=True, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, release=754) 2024-09-21T10:45:52.508 INFO:journalctl@ceph.osd.3.smithi123.stdout:Sep 21 10:45:52 smithi123 podman[75643]: 2024-09-21 10:45:52.50661398 +0000 UTC m=+1.452624139 container cleanup c1672dea7b1fd4a87284f6559bb91234b1dc580559cfeeced7f8179922027e8d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-3, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, architecture=x86_64, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, version=8, vcs-type=git, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870) 2024-09-21T10:45:52.758 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:52 smithi123 ceph-mon[63856]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T10:45:52.758 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:52 smithi123 ceph-mon[63856]: osdmap e121: 8 total, 7 up, 8 in 2024-09-21T10:45:52.758 INFO:journalctl@ceph.osd.3.smithi123.stdout:Sep 21 10:45:52 smithi123 bash[75643]: ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-3 2024-09-21T10:45:52.758 INFO:journalctl@ceph.osd.3.smithi123.stdout:Sep 21 10:45:52 smithi123 podman[75657]: 2024-09-21 10:45:52.725101455 +0000 UTC m=+0.578087277 container remove c1672dea7b1fd4a87284f6559bb91234b1dc580559cfeeced7f8179922027e8d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-3, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, ceph=True, distribution-scope=public, GIT_CLEAN=True, io.openshift.expose-services=, version=8, io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , release=754, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-09-21T10:45:52.838 INFO:teuthology.orchestra.run.smithi123.stdout: 5 215463 10.11 MB/sec execute 413 sec latency 987.926 ms 2024-09-21T10:45:52.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:52 smithi175 ceph-mon[49304]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T10:45:52.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:52 smithi175 ceph-mon[49304]: osdmap e121: 8 total, 7 up, 8 in 2024-09-21T10:45:53.261 INFO:journalctl@ceph.osd.3.smithi123.stdout:Sep 21 10:45:53 smithi123 podman[75757]: 2024-09-21 10:45:53.085440963 +0000 UTC m=+0.025933075 image pull 8f4d013db793795fef09ff5edf96e6b42ba8cd2c9158b2f0b7a5d7c1e657c063 quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472 2024-09-21T10:45:53.261 INFO:journalctl@ceph.osd.3.smithi123.stdout:Sep 21 10:45:53 smithi123 podman[75757]: 2024-09-21 10:45:53.218162256 +0000 UTC m=+0.158654356 container create 8cef4f77599bcbb9d30df4dded1fbe4d880dfea7f2f3c6cda5f14f2cc5e6c553 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-3-deactivate, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-09-21T10:45:53.771 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:53 smithi123 ceph-mon[63856]: pgmap v173: 97 pgs: 16 stale+active+clean, 81 active+clean; 351 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.4 MiB/s wr, 553 op/s 2024-09-21T10:45:53.772 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:53 smithi123 ceph-mon[63856]: osdmap e122: 8 total, 7 up, 8 in 2024-09-21T10:45:53.772 INFO:journalctl@ceph.osd.3.smithi123.stdout:Sep 21 10:45:53 smithi123 podman[75757]: 2024-09-21 10:45:53.602220027 +0000 UTC m=+0.542712141 container init 8cef4f77599bcbb9d30df4dded1fbe4d880dfea7f2f3c6cda5f14f2cc5e6c553 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-3-deactivate, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , ceph=True) 2024-09-21T10:45:53.772 INFO:journalctl@ceph.osd.3.smithi123.stdout:Sep 21 10:45:53 smithi123 podman[75757]: 2024-09-21 10:45:53.60908883 +0000 UTC m=+0.549580937 container start 8cef4f77599bcbb9d30df4dded1fbe4d880dfea7f2f3c6cda5f14f2cc5e6c553 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-3-deactivate, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=) 2024-09-21T10:45:53.838 INFO:teuthology.orchestra.run.smithi123.stdout: 5 216139 10.14 MB/sec execute 414 sec latency 26.261 ms 2024-09-21T10:45:53.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:53 smithi175 ceph-mon[49304]: pgmap v173: 97 pgs: 16 stale+active+clean, 81 active+clean; 351 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.4 MiB/s wr, 553 op/s 2024-09-21T10:45:53.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:53 smithi175 ceph-mon[49304]: osdmap e122: 8 total, 7 up, 8 in 2024-09-21T10:45:54.043 INFO:journalctl@ceph.osd.3.smithi123.stdout:Sep 21 10:45:53 smithi123 podman[75757]: 2024-09-21 10:45:53.797489246 +0000 UTC m=+0.737981360 container attach 8cef4f77599bcbb9d30df4dded1fbe4d880dfea7f2f3c6cda5f14f2cc5e6c553 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-3-deactivate, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3) 2024-09-21T10:45:54.043 INFO:journalctl@ceph.osd.3.smithi123.stdout:Sep 21 10:45:53 smithi123 podman[75757]: 2024-09-21 10:45:53.960657738 +0000 UTC m=+0.901149844 container died 8cef4f77599bcbb9d30df4dded1fbe4d880dfea7f2f3c6cda5f14f2cc5e6c553 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-3-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T10:45:54.836 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:54 smithi123 ceph-mon[63856]: Health check failed: Degraded data redundancy: 145/1611 objects degraded (9.001%), 20 pgs degraded (PG_DEGRADED) 2024-09-21T10:45:54.838 INFO:teuthology.orchestra.run.smithi123.stdout: 5 216386 10.13 MB/sec execute 415 sec latency 55.050 ms 2024-09-21T10:45:54.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:54 smithi175 ceph-mon[49304]: Health check failed: Degraded data redundancy: 145/1611 objects degraded (9.001%), 20 pgs degraded (PG_DEGRADED) 2024-09-21T10:45:55.087 INFO:journalctl@ceph.osd.3.smithi123.stdout:Sep 21 10:45:54 smithi123 podman[75783]: 2024-09-21 10:45:54.887375237 +0000 UTC m=+0.916415851 container remove 8cef4f77599bcbb9d30df4dded1fbe4d880dfea7f2f3c6cda5f14f2cc5e6c553 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-3-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T10:45:55.087 INFO:journalctl@ceph.osd.3.smithi123.stdout:Sep 21 10:45:54 smithi123 systemd[1]: ceph-3ccd15e6-7804-11ef-baf6-efdc52797490@osd.3.service: Deactivated successfully. 2024-09-21T10:45:55.087 INFO:journalctl@ceph.osd.3.smithi123.stdout:Sep 21 10:45:54 smithi123 systemd[1]: Stopped Ceph osd.3 for 3ccd15e6-7804-11ef-baf6-efdc52797490. 2024-09-21T10:45:55.087 INFO:journalctl@ceph.osd.3.smithi123.stdout:Sep 21 10:45:54 smithi123 systemd[1]: ceph-3ccd15e6-7804-11ef-baf6-efdc52797490@osd.3.service: Consumed 4min 10.537s CPU time. 2024-09-21T10:45:55.339 INFO:journalctl@ceph.osd.3.smithi123.stdout:Sep 21 10:45:55 smithi123 systemd[1]: Starting Ceph osd.3 for 3ccd15e6-7804-11ef-baf6-efdc52797490... 2024-09-21T10:45:55.590 INFO:journalctl@ceph.osd.3.smithi123.stdout:Sep 21 10:45:55 smithi123 podman[75878]: 2024-09-21 10:45:55.396499296 +0000 UTC m=+0.023403488 image pull 8f4d013db793795fef09ff5edf96e6b42ba8cd2c9158b2f0b7a5d7c1e657c063 quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472 2024-09-21T10:45:55.590 INFO:journalctl@ceph.osd.3.smithi123.stdout:Sep 21 10:45:55 smithi123 podman[75878]: 2024-09-21 10:45:55.513506803 +0000 UTC m=+0.140410992 container create 3cb297fe4bb21b61352665934353568b56e66ecd477bca9e3289c046e39e5cf4 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-3-activate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-09-21T10:45:55.838 INFO:teuthology.orchestra.run.smithi123.stdout: 5 216650 10.13 MB/sec execute 416 sec latency 91.178 ms 2024-09-21T10:45:55.849 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:55 smithi123 ceph-mon[63856]: pgmap v175: 97 pgs: 12 active+undersized, 3 stale+active+clean, 20 active+undersized+degraded, 62 active+clean; 350 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.9 MiB/s wr, 620 op/s; 145/1611 objects degraded (9.001%) 2024-09-21T10:45:55.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:55 smithi175 ceph-mon[49304]: pgmap v175: 97 pgs: 12 active+undersized, 3 stale+active+clean, 20 active+undersized+degraded, 62 active+clean; 350 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.9 MiB/s wr, 620 op/s; 145/1611 objects degraded (9.001%) 2024-09-21T10:45:56.354 INFO:journalctl@ceph.osd.3.smithi123.stdout:Sep 21 10:45:56 smithi123 podman[75878]: 2024-09-21 10:45:56.183444313 +0000 UTC m=+0.810348504 container init 3cb297fe4bb21b61352665934353568b56e66ecd477bca9e3289c046e39e5cf4 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-3-activate, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T10:45:56.354 INFO:journalctl@ceph.osd.3.smithi123.stdout:Sep 21 10:45:56 smithi123 podman[75878]: 2024-09-21 10:45:56.198817006 +0000 UTC m=+0.825721198 container start 3cb297fe4bb21b61352665934353568b56e66ecd477bca9e3289c046e39e5cf4 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-3-activate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.build-date=20240731) 2024-09-21T10:45:56.607 INFO:journalctl@ceph.osd.3.smithi123.stdout:Sep 21 10:45:56 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-3-activate[75893]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:45:56.607 INFO:journalctl@ceph.osd.3.smithi123.stdout:Sep 21 10:45:56 smithi123 bash[75878]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:45:56.608 INFO:journalctl@ceph.osd.3.smithi123.stdout:Sep 21 10:45:56 smithi123 podman[75878]: 2024-09-21 10:45:56.376145326 +0000 UTC m=+1.003049513 container attach 3cb297fe4bb21b61352665934353568b56e66ecd477bca9e3289c046e39e5cf4 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-3-activate, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T10:45:56.608 INFO:journalctl@ceph.osd.3.smithi123.stdout:Sep 21 10:45:56 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-3-activate[75893]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:45:56.608 INFO:journalctl@ceph.osd.3.smithi123.stdout:Sep 21 10:45:56 smithi123 bash[75878]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:45:56.838 INFO:teuthology.orchestra.run.smithi123.stdout: 5 216933 10.12 MB/sec execute 417 sec latency 58.990 ms 2024-09-21T10:45:57.650 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:57 smithi123 ceph-mon[63856]: pgmap v176: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 351 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.6 MiB/s wr, 521 op/s; 217/1641 objects degraded (13.224%) 2024-09-21T10:45:57.838 INFO:teuthology.orchestra.run.smithi123.stdout: 5 217174 10.11 MB/sec execute 418 sec latency 60.185 ms 2024-09-21T10:45:57.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:57 smithi175 ceph-mon[49304]: pgmap v176: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 351 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.6 MiB/s wr, 521 op/s; 217/1641 objects degraded (13.224%) 2024-09-21T10:45:58.680 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:58 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:45:58.680 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:58 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:45:58.838 INFO:teuthology.orchestra.run.smithi123.stdout: 5 217437 10.10 MB/sec execute 419 sec latency 60.527 ms 2024-09-21T10:45:59.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:58 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:45:59.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:58 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:45:59.693 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:45:59 smithi123 ceph-mon[63856]: pgmap v177: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 350 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 2.6 KiB/s rd, 3.5 MiB/s wr, 304 op/s; 223/1662 objects degraded (13.418%) 2024-09-21T10:45:59.838 INFO:teuthology.orchestra.run.smithi123.stdout: 5 217761 10.09 MB/sec execute 420 sec latency 59.943 ms 2024-09-21T10:46:00.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:45:59 smithi175 ceph-mon[49304]: pgmap v177: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 350 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 2.6 KiB/s rd, 3.5 MiB/s wr, 304 op/s; 223/1662 objects degraded (13.418%) 2024-09-21T10:46:00.698 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:00 smithi123 ceph-mon[63856]: Health check update: Degraded data redundancy: 223/1662 objects degraded (13.418%), 27 pgs degraded (PG_DEGRADED) 2024-09-21T10:46:00.838 INFO:teuthology.orchestra.run.smithi123.stdout: 5 218040 10.08 MB/sec execute 421 sec latency 68.263 ms 2024-09-21T10:46:01.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:00 smithi175 ceph-mon[49304]: Health check update: Degraded data redundancy: 223/1662 objects degraded (13.418%), 27 pgs degraded (PG_DEGRADED) 2024-09-21T10:46:01.201 INFO:journalctl@ceph.osd.3.smithi123.stdout:Sep 21 10:46:00 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-3-activate[75893]: --> Failed to activate via raw: did not find any matching OSD to activate 2024-09-21T10:46:01.201 INFO:journalctl@ceph.osd.3.smithi123.stdout:Sep 21 10:46:00 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-3-activate[75893]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:46:01.201 INFO:journalctl@ceph.osd.3.smithi123.stdout:Sep 21 10:46:00 smithi123 bash[75878]: --> Failed to activate via raw: did not find any matching OSD to activate 2024-09-21T10:46:01.201 INFO:journalctl@ceph.osd.3.smithi123.stdout:Sep 21 10:46:00 smithi123 bash[75878]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:46:01.201 INFO:journalctl@ceph.osd.3.smithi123.stdout:Sep 21 10:46:01 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-3-activate[75893]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:46:01.201 INFO:journalctl@ceph.osd.3.smithi123.stdout:Sep 21 10:46:01 smithi123 bash[75878]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:46:01.201 INFO:journalctl@ceph.osd.3.smithi123.stdout:Sep 21 10:46:01 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-3-activate[75893]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-21T10:46:01.201 INFO:journalctl@ceph.osd.3.smithi123.stdout:Sep 21 10:46:01 smithi123 bash[75878]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-21T10:46:01.452 INFO:journalctl@ceph.osd.3.smithi123.stdout:Sep 21 10:46:01 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-3-activate[75893]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_1 --path /var/lib/ceph/osd/ceph-3 --no-mon-config 2024-09-21T10:46:01.452 INFO:journalctl@ceph.osd.3.smithi123.stdout:Sep 21 10:46:01 smithi123 bash[75878]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_1 --path /var/lib/ceph/osd/ceph-3 --no-mon-config 2024-09-21T10:46:01.703 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:01 smithi123 ceph-mon[63856]: pgmap v178: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 348 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.6 MiB/s wr, 532 op/s; 207/1566 objects degraded (13.218%) 2024-09-21T10:46:01.704 INFO:journalctl@ceph.osd.3.smithi123.stdout:Sep 21 10:46:01 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-3-activate[75893]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-21T10:46:01.704 INFO:journalctl@ceph.osd.3.smithi123.stdout:Sep 21 10:46:01 smithi123 bash[75878]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-21T10:46:01.704 INFO:journalctl@ceph.osd.3.smithi123.stdout:Sep 21 10:46:01 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-3-activate[75893]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-3/block 2024-09-21T10:46:01.704 INFO:journalctl@ceph.osd.3.smithi123.stdout:Sep 21 10:46:01 smithi123 bash[75878]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-3/block 2024-09-21T10:46:01.704 INFO:journalctl@ceph.osd.3.smithi123.stdout:Sep 21 10:46:01 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-3-activate[75893]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-21T10:46:01.704 INFO:journalctl@ceph.osd.3.smithi123.stdout:Sep 21 10:46:01 smithi123 bash[75878]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-21T10:46:01.704 INFO:journalctl@ceph.osd.3.smithi123.stdout:Sep 21 10:46:01 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-3-activate[75893]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-21T10:46:01.704 INFO:journalctl@ceph.osd.3.smithi123.stdout:Sep 21 10:46:01 smithi123 bash[75878]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-21T10:46:01.704 INFO:journalctl@ceph.osd.3.smithi123.stdout:Sep 21 10:46:01 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-3-activate[75893]: --> ceph-volume lvm activate successful for osd ID: 3 2024-09-21T10:46:01.704 INFO:journalctl@ceph.osd.3.smithi123.stdout:Sep 21 10:46:01 smithi123 bash[75878]: --> ceph-volume lvm activate successful for osd ID: 3 2024-09-21T10:46:01.704 INFO:journalctl@ceph.osd.3.smithi123.stdout:Sep 21 10:46:01 smithi123 podman[75878]: 2024-09-21 10:46:01.621920752 +0000 UTC m=+6.248824940 container died 3cb297fe4bb21b61352665934353568b56e66ecd477bca9e3289c046e39e5cf4 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-3-activate, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T10:46:01.838 INFO:teuthology.orchestra.run.smithi123.stdout: 5 218339 10.07 MB/sec execute 422 sec latency 43.317 ms 2024-09-21T10:46:02.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:01 smithi175 ceph-mon[49304]: pgmap v178: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 348 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.6 MiB/s wr, 532 op/s; 207/1566 objects degraded (13.218%) 2024-09-21T10:46:02.838 INFO:teuthology.orchestra.run.smithi123.stdout: 5 218635 10.06 MB/sec execute 423 sec latency 65.518 ms 2024-09-21T10:46:02.974 INFO:journalctl@ceph.osd.3.smithi123.stdout:Sep 21 10:46:02 smithi123 podman[76116]: 2024-09-21 10:46:02.759138096 +0000 UTC m=+1.129450777 container remove 3cb297fe4bb21b61352665934353568b56e66ecd477bca9e3289c046e39e5cf4 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-3-activate, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240731) 2024-09-21T10:46:03.235 INFO:journalctl@ceph.osd.3.smithi123.stdout:Sep 21 10:46:02 smithi123 podman[76171]: 2024-09-21 10:46:02.903954367 +0000 UTC m=+0.022563137 image pull 8f4d013db793795fef09ff5edf96e6b42ba8cd2c9158b2f0b7a5d7c1e657c063 quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472 2024-09-21T10:46:03.235 INFO:journalctl@ceph.osd.3.smithi123.stdout:Sep 21 10:46:03 smithi123 podman[76171]: 2024-09-21 10:46:03.057143535 +0000 UTC m=+0.175752281 container create 7a082fcad8e897d96dd27cc72399d5e23e119699d6490ce9300e777f4eb6ad80 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-3, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, GIT_BRANCH=HEAD) 2024-09-21T10:46:03.489 INFO:journalctl@ceph.osd.3.smithi123.stdout:Sep 21 10:46:03 smithi123 podman[76171]: 2024-09-21 10:46:03.414270743 +0000 UTC m=+0.532879503 container init 7a082fcad8e897d96dd27cc72399d5e23e119699d6490ce9300e777f4eb6ad80 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-3, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T10:46:03.489 INFO:journalctl@ceph.osd.3.smithi123.stdout:Sep 21 10:46:03 smithi123 podman[76171]: 2024-09-21 10:46:03.419666076 +0000 UTC m=+0.538274826 container start 7a082fcad8e897d96dd27cc72399d5e23e119699d6490ce9300e777f4eb6ad80 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-3, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, ceph=True, GIT_BRANCH=HEAD) 2024-09-21T10:46:03.489 INFO:journalctl@ceph.osd.3.smithi123.stdout:Sep 21 10:46:03 smithi123 ceph-osd[76189]: -- 172.21.15.123:0/3760447095 <== mon.0 v2:172.21.15.123:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55d4080e81e0 con 0x55d408074000 2024-09-21T10:46:03.740 INFO:journalctl@ceph.osd.3.smithi123.stdout:Sep 21 10:46:03 smithi123 bash[76171]: 7a082fcad8e897d96dd27cc72399d5e23e119699d6490ce9300e777f4eb6ad80 2024-09-21T10:46:03.740 INFO:journalctl@ceph.osd.3.smithi123.stdout:Sep 21 10:46:03 smithi123 systemd[1]: Started Ceph osd.3 for 3ccd15e6-7804-11ef-baf6-efdc52797490. 2024-09-21T10:46:03.740 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:03 smithi123 ceph-mon[63856]: pgmap v179: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 348 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.9 MiB/s wr, 464 op/s; 207/1566 objects degraded (13.218%) 2024-09-21T10:46:03.740 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:03 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:46:03.741 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:03 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:46:03.741 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:03 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:46:03.838 INFO:teuthology.orchestra.run.smithi123.stdout: 5 219145 10.09 MB/sec execute 424 sec latency 42.085 ms 2024-09-21T10:46:04.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:03 smithi175 ceph-mon[49304]: pgmap v179: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 348 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.9 MiB/s wr, 464 op/s; 207/1566 objects degraded (13.218%) 2024-09-21T10:46:04.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:03 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:46:04.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:03 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:46:04.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:03 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:46:04.838 INFO:teuthology.orchestra.run.smithi123.stdout: 5 219647 10.10 MB/sec execute 425 sec latency 39.862 ms 2024-09-21T10:46:05.779 INFO:journalctl@ceph.osd.3.smithi123.stdout:Sep 21 10:46:05 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-3[76185]: 2024-09-21T10:46:05.748+0000 7ffbfc9dc640 -1 Falling back to public interface 2024-09-21T10:46:05.780 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:05 smithi123 ceph-mon[63856]: pgmap v180: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 342 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.5 MiB/s wr, 574 op/s; 205/1572 objects degraded (13.041%) 2024-09-21T10:46:05.780 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:05 smithi123 ceph-mon[63856]: Health check update: Degraded data redundancy: 205/1572 objects degraded (13.041%), 27 pgs degraded (PG_DEGRADED) 2024-09-21T10:46:05.838 INFO:teuthology.orchestra.run.smithi123.stdout: 5 219897 10.10 MB/sec execute 426 sec latency 35.565 ms 2024-09-21T10:46:06.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:05 smithi175 ceph-mon[49304]: pgmap v180: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 342 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.5 MiB/s wr, 574 op/s; 205/1572 objects degraded (13.041%) 2024-09-21T10:46:06.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:05 smithi175 ceph-mon[49304]: Health check update: Degraded data redundancy: 205/1572 objects degraded (13.041%), 27 pgs degraded (PG_DEGRADED) 2024-09-21T10:46:06.838 INFO:teuthology.orchestra.run.smithi123.stdout: 5 220153 10.09 MB/sec execute 427 sec latency 58.322 ms 2024-09-21T10:46:07.838 INFO:teuthology.orchestra.run.smithi123.stdout: 5 220384 10.07 MB/sec execute 428 sec latency 57.752 ms 2024-09-21T10:46:07.885 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:07 smithi123 ceph-mon[63856]: pgmap v181: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 345 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.4 MiB/s wr, 413 op/s; 212/1593 objects degraded (13.308%) 2024-09-21T10:46:08.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:07 smithi175 ceph-mon[49304]: pgmap v181: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 345 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.4 MiB/s wr, 413 op/s; 212/1593 objects degraded (13.308%) 2024-09-21T10:46:08.838 INFO:teuthology.orchestra.run.smithi123.stdout: 5 220547 10.06 MB/sec execute 429 sec latency 65.927 ms 2024-09-21T10:46:09.838 INFO:teuthology.orchestra.run.smithi123.stdout: 5 220792 10.06 MB/sec execute 430 sec latency 86.026 ms 2024-09-21T10:46:09.925 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:09 smithi123 ceph-mon[63856]: pgmap v182: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 346 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.3 MiB/s wr, 409 op/s; 212/1584 objects degraded (13.384%) 2024-09-21T10:46:10.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:09 smithi175 ceph-mon[49304]: pgmap v182: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 346 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.3 MiB/s wr, 409 op/s; 212/1584 objects degraded (13.384%) 2024-09-21T10:46:10.725 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:10 smithi123 ceph-mon[63856]: Health check update: Degraded data redundancy: 212/1584 objects degraded (13.384%), 27 pgs degraded (PG_DEGRADED) 2024-09-21T10:46:10.838 INFO:teuthology.orchestra.run.smithi123.stdout: 5 221165 10.05 MB/sec execute 431 sec latency 49.421 ms 2024-09-21T10:46:11.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:10 smithi175 ceph-mon[49304]: Health check update: Degraded data redundancy: 212/1584 objects degraded (13.384%), 27 pgs degraded (PG_DEGRADED) 2024-09-21T10:46:11.738 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:11 smithi123 ceph-mon[63856]: pgmap v183: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 352 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 5.6 MiB/s wr, 562 op/s; 216/1635 objects degraded (13.211%) 2024-09-21T10:46:11.838 INFO:teuthology.orchestra.run.smithi123.stdout: 5 221528 10.04 MB/sec execute 432 sec latency 39.900 ms 2024-09-21T10:46:12.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:11 smithi175 ceph-mon[49304]: pgmap v183: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 352 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 5.6 MiB/s wr, 562 op/s; 216/1635 objects degraded (13.211%) 2024-09-21T10:46:12.839 INFO:teuthology.orchestra.run.smithi123.stdout: 5 221870 10.03 MB/sec execute 433 sec latency 57.995 ms 2024-09-21T10:46:13.520 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:13 smithi123 ceph-mon[63856]: pgmap v184: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 352 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.9 MiB/s wr, 377 op/s; 216/1635 objects degraded (13.211%) 2024-09-21T10:46:13.521 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:13 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:46:13.521 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:13 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:46:13.521 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:13 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:46:13.521 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:13 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:46:13.521 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:13 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:46:13.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:13 smithi175 ceph-mon[49304]: pgmap v184: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 352 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.9 MiB/s wr, 377 op/s; 216/1635 objects degraded (13.211%) 2024-09-21T10:46:13.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:13 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:46:13.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:13 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:46:13.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:13 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:46:13.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:13 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:46:13.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:13 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:46:13.838 INFO:teuthology.orchestra.run.smithi123.stdout: 5 222272 10.03 MB/sec execute 434 sec latency 40.671 ms 2024-09-21T10:46:14.023 INFO:journalctl@ceph.osd.3.smithi123.stdout:Sep 21 10:46:13 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-3[76185]: 2024-09-21T10:46:13.938+0000 7ffbfc9dc640 -1 osd.3 0 read_superblock omap replica is missing. 2024-09-21T10:46:14.839 INFO:teuthology.orchestra.run.smithi123.stdout: 5 222609 10.04 MB/sec execute 435 sec latency 44.701 ms 2024-09-21T10:46:15.350 INFO:journalctl@ceph.osd.3.smithi123.stdout:Sep 21 10:46:15 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-3[76185]: 2024-09-21T10:46:15.102+0000 7ffbfc9dc640 -1 osd.3 120 log_to_monitors true 2024-09-21T10:46:15.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:15 smithi123 ceph-mon[63856]: pgmap v185: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 350 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.3 MiB/s wr, 564 op/s; 215/1623 objects degraded (13.247%) 2024-09-21T10:46:15.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:15 smithi123 ceph-mon[63856]: Health check update: Degraded data redundancy: 215/1623 objects degraded (13.247%), 27 pgs degraded (PG_DEGRADED) 2024-09-21T10:46:15.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:15 smithi123 ceph-mon[63856]: from='osd.3 [v2:172.21.15.123:6826/1080599128,v1:172.21.15.123:6827/1080599128]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-21T10:46:15.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:15 smithi175 ceph-mon[49304]: pgmap v185: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 350 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.3 MiB/s wr, 564 op/s; 215/1623 objects degraded (13.247%) 2024-09-21T10:46:15.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:15 smithi175 ceph-mon[49304]: Health check update: Degraded data redundancy: 215/1623 objects degraded (13.247%), 27 pgs degraded (PG_DEGRADED) 2024-09-21T10:46:15.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:15 smithi175 ceph-mon[49304]: from='osd.3 [v2:172.21.15.123:6826/1080599128,v1:172.21.15.123:6827/1080599128]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-21T10:46:15.839 INFO:teuthology.orchestra.run.smithi123.stdout: 5 222883 10.04 MB/sec execute 436 sec latency 185.937 ms 2024-09-21T10:46:15.878 INFO:journalctl@ceph.osd.3.smithi123.stdout:Sep 21 10:46:15 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-3[76185]: 2024-09-21T10:46:15.830+0000 7ffbf077e640 -1 osd.3 120 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T10:46:16.381 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:16 smithi123 ceph-mon[63856]: from='osd.3 [v2:172.21.15.123:6826/1080599128,v1:172.21.15.123:6827/1080599128]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-21T10:46:16.381 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:16 smithi123 ceph-mon[63856]: osdmap e123: 8 total, 7 up, 8 in 2024-09-21T10:46:16.381 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:16 smithi123 ceph-mon[63856]: from='osd.3 [v2:172.21.15.123:6826/1080599128,v1:172.21.15.123:6827/1080599128]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi123", "root=default"]} : dispatch 2024-09-21T10:46:16.382 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:16 smithi123 ceph-mon[63856]: from='osd.3 [v2:172.21.15.123:6826/1080599128,v1:172.21.15.123:6827/1080599128]' entity='osd.3' 2024-09-21T10:46:16.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:16 smithi175 ceph-mon[49304]: from='osd.3 [v2:172.21.15.123:6826/1080599128,v1:172.21.15.123:6827/1080599128]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-21T10:46:16.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:16 smithi175 ceph-mon[49304]: osdmap e123: 8 total, 7 up, 8 in 2024-09-21T10:46:16.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:16 smithi175 ceph-mon[49304]: from='osd.3 [v2:172.21.15.123:6826/1080599128,v1:172.21.15.123:6827/1080599128]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi123", "root=default"]} : dispatch 2024-09-21T10:46:16.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:16 smithi175 ceph-mon[49304]: from='osd.3 [v2:172.21.15.123:6826/1080599128,v1:172.21.15.123:6827/1080599128]' entity='osd.3' 2024-09-21T10:46:16.839 INFO:teuthology.orchestra.run.smithi123.stdout: 5 223413 10.06 MB/sec execute 437 sec latency 39.086 ms 2024-09-21T10:46:17.387 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:17 smithi123 ceph-mon[63856]: pgmap v187: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 349 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.6 MiB/s wr, 471 op/s; 214/1623 objects degraded (13.185%) 2024-09-21T10:46:17.387 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:17 smithi123 ceph-mon[63856]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T10:46:17.387 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:17 smithi123 ceph-mon[63856]: osd.3 [v2:172.21.15.123:6826/1080599128,v1:172.21.15.123:6827/1080599128] boot 2024-09-21T10:46:17.387 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:17 smithi123 ceph-mon[63856]: osdmap e124: 8 total, 8 up, 8 in 2024-09-21T10:46:17.387 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:17 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-21T10:46:17.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:17 smithi175 ceph-mon[49304]: pgmap v187: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 349 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.6 MiB/s wr, 471 op/s; 214/1623 objects degraded (13.185%) 2024-09-21T10:46:17.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:17 smithi175 ceph-mon[49304]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T10:46:17.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:17 smithi175 ceph-mon[49304]: osd.3 [v2:172.21.15.123:6826/1080599128,v1:172.21.15.123:6827/1080599128] boot 2024-09-21T10:46:17.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:17 smithi175 ceph-mon[49304]: osdmap e124: 8 total, 8 up, 8 in 2024-09-21T10:46:17.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:17 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-21T10:46:17.839 INFO:teuthology.orchestra.run.smithi123.stdout: 5 223585 10.04 MB/sec execute 438 sec latency 1039.147 ms 2024-09-21T10:46:18.839 INFO:teuthology.orchestra.run.smithi123.stdout: 5 223654 10.01 MB/sec execute 439 sec latency 2039.179 ms 2024-09-21T10:46:18.936 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:18 smithi123 ceph-mon[63856]: osdmap e125: 8 total, 8 up, 8 in 2024-09-21T10:46:18.936 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:18 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:46:18.936 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:18 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:46:18.937 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:18 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:46:18.937 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:18 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:46:18.937 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:18 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:46:18.937 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:18 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:46:19.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:18 smithi175 ceph-mon[49304]: osdmap e125: 8 total, 8 up, 8 in 2024-09-21T10:46:19.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:18 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:46:19.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:18 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:46:19.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:18 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:46:19.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:18 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:46:19.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:18 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:46:19.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:18 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:46:19.839 INFO:teuthology.orchestra.run.smithi123.stdout: 5 223980 10.02 MB/sec execute 440 sec latency 2130.586 ms 2024-09-21T10:46:19.955 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:19 smithi123 ceph-mon[63856]: pgmap v190: 97 pgs: 2 remapped+peering, 8 active+undersized, 21 active+undersized+degraded, 66 active+clean; 356 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 5.4 MiB/s wr, 680 op/s; 174/1644 objects degraded (10.584%); 14 B/s, 0 objects/s recovering 2024-09-21T10:46:19.955 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:19 smithi123 ceph-mon[63856]: osdmap e126: 8 total, 8 up, 8 in 2024-09-21T10:46:19.955 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:19 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:46:19.956 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:19 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:46:19.956 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:19 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:46:19.956 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:19 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:46:19.956 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:19 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-21T10:46:20.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:19 smithi175 ceph-mon[49304]: pgmap v190: 97 pgs: 2 remapped+peering, 8 active+undersized, 21 active+undersized+degraded, 66 active+clean; 356 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 5.4 MiB/s wr, 680 op/s; 174/1644 objects degraded (10.584%); 14 B/s, 0 objects/s recovering 2024-09-21T10:46:20.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:19 smithi175 ceph-mon[49304]: osdmap e126: 8 total, 8 up, 8 in 2024-09-21T10:46:20.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:19 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:46:20.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:19 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:46:20.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:19 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:46:20.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:19 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:46:20.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:19 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-21T10:46:20.830 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:20 smithi123 ceph-mon[63856]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-21T10:46:20.830 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:20 smithi123 ceph-mon[63856]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-09-21T10:46:20.830 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:20 smithi123 ceph-mon[63856]: osdmap e127: 8 total, 8 up, 8 in 2024-09-21T10:46:20.830 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:20 smithi123 ceph-mon[63856]: Health check update: Degraded data redundancy: 174/1644 objects degraded (10.584%), 21 pgs degraded (PG_DEGRADED) 2024-09-21T10:46:20.839 INFO:teuthology.orchestra.run.smithi123.stdout: 5 224199 10.01 MB/sec execute 441 sec latency 60.441 ms 2024-09-21T10:46:20.888 INFO:teuthology.orchestra.run.smithi123.stdout:true 2024-09-21T10:46:21.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:20 smithi175 ceph-mon[49304]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-21T10:46:21.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:20 smithi175 ceph-mon[49304]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-09-21T10:46:21.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:20 smithi175 ceph-mon[49304]: osdmap e127: 8 total, 8 up, 8 in 2024-09-21T10:46:21.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:20 smithi175 ceph-mon[49304]: Health check update: Degraded data redundancy: 174/1644 objects degraded (10.584%), 21 pgs degraded (PG_DEGRADED) 2024-09-21T10:46:21.839 INFO:teuthology.orchestra.run.smithi123.stdout: 5 224421 10.01 MB/sec execute 442 sec latency 790.250 ms 2024-09-21T10:46:21.859 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:21 smithi123 ceph-mon[63856]: pgmap v193: 97 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 7 remapped+peering, 4 active+undersized, 6 active+undersized+degraded, 75 active+clean; 353 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.8 MiB/s wr, 528 op/s; 158/1647 objects degraded (9.593%); 54 KiB/s, 3 objects/s recovering 2024-09-21T10:46:21.859 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:21 smithi123 ceph-mon[63856]: osdmap e128: 8 total, 8 up, 8 in 2024-09-21T10:46:21.898 INFO:teuthology.orchestra.run.smithi123.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T10:46:21.899 INFO:teuthology.orchestra.run.smithi123.stdout:alertmanager.smithi123 smithi123 *:9093,9094 running (6m) 9s ago 16m 22.0M - 0.27.0 11f11916f8cd 5d550bd97de5 2024-09-21T10:46:21.899 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi123 smithi123 running (3m) 9s ago 15m 6899k - 19.3.0-5103-g5c1cdc3c 8f4d013db793 c5e1ef39a0a8 2024-09-21T10:46:21.899 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi175 smithi175 running (3m) 3m ago 15m 6895k - 19.3.0-5103-g5c1cdc3c 8f4d013db793 fb1ca999ccfd 2024-09-21T10:46:21.899 INFO:teuthology.orchestra.run.smithi123.stdout:grafana.smithi123 smithi123 *:3000 running (6m) 9s ago 15m 52.6M - dad864ee21e9 771df81763ba 2024-09-21T10:46:21.899 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi123.uqoyow smithi123 running (9m) 9s ago 9m 41.2M - 17.2.0 e1d6a67b021e f0a21eaac266 2024-09-21T10:46:21.899 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi175.jrlevw smithi175 running (9m) 3m ago 9m 13.5M - 17.2.0 e1d6a67b021e b21b16f4bc50 2024-09-21T10:46:21.899 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi123.stnuit smithi123 *:8443,9283,8765 running (5m) 9s ago 17m 519M - 19.3.0-5103-g5c1cdc3c 8f4d013db793 6812cc5cb56b 2024-09-21T10:46:21.899 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi175.txvwrq smithi175 *:8443,9283,8765 running (5m) 3m ago 15m 453M - 19.3.0-5103-g5c1cdc3c 8f4d013db793 304f0408075e 2024-09-21T10:46:21.899 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi123 smithi123 running (4m) 9s ago 17m 52.0M 2048M 19.3.0-5103-g5c1cdc3c 8f4d013db793 6b95207a2d80 2024-09-21T10:46:21.900 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi175 smithi175 running (4m) 3m ago 15m 36.7M 2048M 19.3.0-5103-g5c1cdc3c 8f4d013db793 3abee2d37f0f 2024-09-21T10:46:21.900 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.0.0.smithi123.ddatpr smithi123 *:2049 running (9m) 9s ago 9m 281M - 4.0 e1d6a67b021e 36f8cb2281eb 2024-09-21T10:46:21.900 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.1.0.smithi175.wpawer smithi175 *:2049 running (9m) 3m ago 9m 90.3M - 4.0 e1d6a67b021e 2200aeed9200 2024-09-21T10:46:21.900 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi123 smithi123 *:9100 running (6m) 9s ago 15m 12.3M - 1.7.0 72c9c2088986 ca7d0b29af81 2024-09-21T10:46:21.900 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi175 smithi175 *:9100 running (6m) 3m ago 15m 11.1M - 1.7.0 72c9c2088986 46c4f6b7c149 2024-09-21T10:46:21.900 INFO:teuthology.orchestra.run.smithi123.stdout:osd.0 smithi123 running (2m) 9s ago 14m 362M 1936M 19.3.0-5103-g5c1cdc3c 8f4d013db793 cc40f695c3a8 2024-09-21T10:46:21.900 INFO:teuthology.orchestra.run.smithi123.stdout:osd.1 smithi123 running (2m) 9s ago 13m 279M 1936M 19.3.0-5103-g5c1cdc3c 8f4d013db793 1be249025c7f 2024-09-21T10:46:21.900 INFO:teuthology.orchestra.run.smithi123.stdout:osd.2 smithi123 running (68s) 9s ago 13m 232M 1936M 19.3.0-5103-g5c1cdc3c 8f4d013db793 647aec759e3c 2024-09-21T10:46:21.900 INFO:teuthology.orchestra.run.smithi123.stdout:osd.3 smithi123 running (18s) 9s ago 12m 11.6M 1936M 19.3.0-5103-g5c1cdc3c 8f4d013db793 7a082fcad8e8 2024-09-21T10:46:21.900 INFO:teuthology.orchestra.run.smithi123.stdout:osd.4 smithi175 running (12m) 3m ago 12m 498M 2704M 17.2.0 e1d6a67b021e 48fe728c259e 2024-09-21T10:46:21.900 INFO:teuthology.orchestra.run.smithi123.stdout:osd.5 smithi175 running (12m) 3m ago 12m 331M 2704M 17.2.0 e1d6a67b021e e270d5a21b14 2024-09-21T10:46:21.900 INFO:teuthology.orchestra.run.smithi123.stdout:osd.6 smithi175 running (11m) 3m ago 11m 517M 2704M 17.2.0 e1d6a67b021e 9bcbf018eb20 2024-09-21T10:46:21.901 INFO:teuthology.orchestra.run.smithi123.stdout:osd.7 smithi175 running (11m) 3m ago 11m 383M 2704M 17.2.0 e1d6a67b021e e9b8f1f90994 2024-09-21T10:46:21.901 INFO:teuthology.orchestra.run.smithi123.stdout:prometheus.smithi123 smithi123 *:9095 running (5m) 9s ago 14m 55.4M - 2.51.0 1d3b7f56885b 37f944c9de88 2024-09-21T10:46:22.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:21 smithi175 ceph-mon[49304]: pgmap v193: 97 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 7 remapped+peering, 4 active+undersized, 6 active+undersized+degraded, 75 active+clean; 353 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.8 MiB/s wr, 528 op/s; 158/1647 objects degraded (9.593%); 54 KiB/s, 3 objects/s recovering 2024-09-21T10:46:22.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:21 smithi175 ceph-mon[49304]: osdmap e128: 8 total, 8 up, 8 in 2024-09-21T10:46:22.504 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-09-21T10:46:22.504 INFO:teuthology.orchestra.run.smithi123.stdout: "mon": { 2024-09-21T10:46:22.504 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev)": 2 2024-09-21T10:46:22.504 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:46:22.504 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr": { 2024-09-21T10:46:22.505 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev)": 2 2024-09-21T10:46:22.505 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:46:22.505 INFO:teuthology.orchestra.run.smithi123.stdout: "osd": { 2024-09-21T10:46:22.505 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-09-21T10:46:22.505 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev)": 4 2024-09-21T10:46:22.505 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:46:22.505 INFO:teuthology.orchestra.run.smithi123.stdout: "mds": { 2024-09-21T10:46:22.505 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T10:46:22.505 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:46:22.505 INFO:teuthology.orchestra.run.smithi123.stdout: "overall": { 2024-09-21T10:46:22.505 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-09-21T10:46:22.505 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev)": 8 2024-09-21T10:46:22.505 INFO:teuthology.orchestra.run.smithi123.stdout: } 2024-09-21T10:46:22.505 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-09-21T10:46:22.841 INFO:teuthology.orchestra.run.smithi123.stdout: 5 224633 9.99 MB/sec execute 443 sec latency 800.160 ms 2024-09-21T10:46:22.885 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:22 smithi123 ceph-mon[63856]: from='client.34332 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:46:22.885 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:22 smithi123 ceph-mon[63856]: from='client.34336 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:46:22.885 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:22 smithi123 ceph-mon[63856]: osdmap e129: 8 total, 8 up, 8 in 2024-09-21T10:46:22.885 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:22 smithi123 ceph-mon[63856]: from='client.34340 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:46:22.885 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:22 smithi123 ceph-mon[63856]: pgmap v196: 97 pgs: 1 peering, 5 active+recovery_wait+undersized+degraded+remapped, 6 remapped+peering, 85 active+clean; 349 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 3.2 MiB/s wr, 141 op/s; 146/1635 objects degraded (8.930%); 89 KiB/s, 11 objects/s recovering 2024-09-21T10:46:22.885 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:22 smithi123 ceph-mon[63856]: from='client.? 172.21.15.123:0/2031331110' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:46:22.885 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:22 smithi123 ceph-mon[63856]: osdmap e130: 8 total, 8 up, 8 in 2024-09-21T10:46:22.922 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-09-21T10:46:22.922 INFO:teuthology.orchestra.run.smithi123.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472", 2024-09-21T10:46:22.923 INFO:teuthology.orchestra.run.smithi123.stdout: "in_progress": true, 2024-09-21T10:46:22.923 INFO:teuthology.orchestra.run.smithi123.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T10:46:22.923 INFO:teuthology.orchestra.run.smithi123.stdout: "services_complete": [ 2024-09-21T10:46:22.923 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr", 2024-09-21T10:46:22.923 INFO:teuthology.orchestra.run.smithi123.stdout: "crash", 2024-09-21T10:46:22.923 INFO:teuthology.orchestra.run.smithi123.stdout: "mon" 2024-09-21T10:46:22.923 INFO:teuthology.orchestra.run.smithi123.stdout: ], 2024-09-21T10:46:22.923 INFO:teuthology.orchestra.run.smithi123.stdout: "progress": "10/23 daemons upgraded", 2024-09-21T10:46:22.923 INFO:teuthology.orchestra.run.smithi123.stdout: "message": "Currently upgrading osd daemons", 2024-09-21T10:46:22.923 INFO:teuthology.orchestra.run.smithi123.stdout: "is_paused": false 2024-09-21T10:46:22.923 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-09-21T10:46:23.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:22 smithi175 ceph-mon[49304]: from='client.34332 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:46:23.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:22 smithi175 ceph-mon[49304]: from='client.34336 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:46:23.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:22 smithi175 ceph-mon[49304]: osdmap e129: 8 total, 8 up, 8 in 2024-09-21T10:46:23.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:22 smithi175 ceph-mon[49304]: from='client.34340 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:46:23.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:22 smithi175 ceph-mon[49304]: pgmap v196: 97 pgs: 1 peering, 5 active+recovery_wait+undersized+degraded+remapped, 6 remapped+peering, 85 active+clean; 349 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 3.2 MiB/s wr, 141 op/s; 146/1635 objects degraded (8.930%); 89 KiB/s, 11 objects/s recovering 2024-09-21T10:46:23.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:22 smithi175 ceph-mon[49304]: from='client.? 172.21.15.123:0/2031331110' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:46:23.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:22 smithi175 ceph-mon[49304]: osdmap e130: 8 total, 8 up, 8 in 2024-09-21T10:46:23.466 INFO:teuthology.orchestra.run.smithi123.stdout:HEALTH_WARN Degraded data redundancy: 146/1635 objects degraded (8.930%), 5 pgs degraded 2024-09-21T10:46:23.466 INFO:teuthology.orchestra.run.smithi123.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 146/1635 objects degraded (8.930%), 5 pgs degraded 2024-09-21T10:46:23.466 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.7 is active+recovery_wait+undersized+degraded+remapped, acting [0,7] 2024-09-21T10:46:23.466 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.b is active+recovery_wait+undersized+degraded+remapped, acting [0,4] 2024-09-21T10:46:23.467 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.12 is active+recovery_wait+undersized+degraded+remapped, acting [0,7] 2024-09-21T10:46:23.467 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.17 is active+recovery_wait+undersized+degraded+remapped, acting [0,5] 2024-09-21T10:46:23.467 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.18 is active+recovery_wait+undersized+degraded+remapped, acting [0,1] 2024-09-21T10:46:23.839 INFO:teuthology.orchestra.run.smithi123.stdout: 5 224853 9.98 MB/sec execute 444 sec latency 796.095 ms 2024-09-21T10:46:23.923 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:23 smithi123 ceph-mon[63856]: from='client.34348 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:46:23.923 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:23 smithi123 ceph-mon[63856]: from='client.? 172.21.15.123:0/3461546622' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T10:46:24.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:23 smithi175 ceph-mon[49304]: from='client.34348 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:46:24.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:23 smithi175 ceph-mon[49304]: from='client.? 172.21.15.123:0/3461546622' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T10:46:24.839 INFO:teuthology.orchestra.run.smithi123.stdout: 5 225009 9.96 MB/sec execute 445 sec latency 994.167 ms 2024-09-21T10:46:25.098 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:24 smithi123 ceph-mon[63856]: osdmap e131: 8 total, 8 up, 8 in 2024-09-21T10:46:25.098 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:24 smithi123 ceph-mon[63856]: pgmap v199: 97 pgs: 1 active+undersized+remapped, 1 peering, 5 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 89 active+clean; 350 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 3.5 MiB/s wr, 419 op/s; 146/1581 objects degraded (9.235%); 37 KiB/s, 42 objects/s recovering 2024-09-21T10:46:25.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:24 smithi175 ceph-mon[49304]: osdmap e131: 8 total, 8 up, 8 in 2024-09-21T10:46:25.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:24 smithi175 ceph-mon[49304]: pgmap v199: 97 pgs: 1 active+undersized+remapped, 1 peering, 5 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 89 active+clean; 350 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 3.5 MiB/s wr, 419 op/s; 146/1581 objects degraded (9.235%); 37 KiB/s, 42 objects/s recovering 2024-09-21T10:46:25.839 INFO:teuthology.orchestra.run.smithi123.stdout: 5 225339 9.95 MB/sec execute 446 sec latency 1026.810 ms 2024-09-21T10:46:25.867 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:25 smithi123 ceph-mon[63856]: osdmap e132: 8 total, 8 up, 8 in 2024-09-21T10:46:25.867 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:25 smithi123 ceph-mon[63856]: Health check update: Degraded data redundancy: 146/1581 objects degraded (9.235%), 5 pgs degraded (PG_DEGRADED) 2024-09-21T10:46:26.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:25 smithi175 ceph-mon[49304]: osdmap e132: 8 total, 8 up, 8 in 2024-09-21T10:46:26.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:25 smithi175 ceph-mon[49304]: Health check update: Degraded data redundancy: 146/1581 objects degraded (9.235%), 5 pgs degraded (PG_DEGRADED) 2024-09-21T10:46:26.839 INFO:teuthology.orchestra.run.smithi123.stdout: 5 225699 9.95 MB/sec execute 447 sec latency 40.476 ms 2024-09-21T10:46:26.889 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:26 smithi123 ceph-mon[63856]: pgmap v201: 97 pgs: 1 peering, 96 active+clean; 348 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 3.7 MiB/s wr, 416 op/s; 533 KiB/s, 71 objects/s recovering 2024-09-21T10:46:27.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:26 smithi175 ceph-mon[49304]: pgmap v201: 97 pgs: 1 peering, 96 active+clean; 348 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 3.7 MiB/s wr, 416 op/s; 533 KiB/s, 71 objects/s recovering 2024-09-21T10:46:27.839 INFO:teuthology.orchestra.run.smithi123.stdout: 5 226153 9.97 MB/sec execute 448 sec latency 46.335 ms 2024-09-21T10:46:27.900 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:27 smithi123 ceph-mon[63856]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 146/1581 objects degraded (9.235%), 5 pgs degraded) 2024-09-21T10:46:27.900 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:27 smithi123 ceph-mon[63856]: Cluster is now healthy 2024-09-21T10:46:27.900 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:27 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:46:27.900 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:27 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:46:28.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:27 smithi175 ceph-mon[49304]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 146/1581 objects degraded (9.235%), 5 pgs degraded) 2024-09-21T10:46:28.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:27 smithi175 ceph-mon[49304]: Cluster is now healthy 2024-09-21T10:46:28.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:27 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:46:28.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:27 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:46:28.839 INFO:teuthology.orchestra.run.smithi123.stdout: 5 226532 9.96 MB/sec execute 449 sec latency 52.072 ms 2024-09-21T10:46:28.919 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:28 smithi123 ceph-mon[63856]: pgmap v202: 97 pgs: 1 peering, 96 active+clean; 351 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.4 MiB/s wr, 338 op/s; 395 KiB/s, 52 objects/s recovering 2024-09-21T10:46:29.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:28 smithi175 ceph-mon[49304]: pgmap v202: 97 pgs: 1 peering, 96 active+clean; 351 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.4 MiB/s wr, 338 op/s; 395 KiB/s, 52 objects/s recovering 2024-09-21T10:46:29.839 INFO:teuthology.orchestra.run.smithi123.stdout: 5 226856 9.97 MB/sec execute 450 sec latency 56.917 ms 2024-09-21T10:46:30.839 INFO:teuthology.orchestra.run.smithi123.stdout: 5 227167 9.97 MB/sec execute 451 sec latency 79.914 ms 2024-09-21T10:46:31.470 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:31 smithi123 ceph-mon[63856]: pgmap v203: 97 pgs: 97 active+clean; 350 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 4.6 MiB/s wr, 404 op/s; 318 KiB/s, 42 objects/s recovering 2024-09-21T10:46:31.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:31 smithi175 ceph-mon[49304]: pgmap v203: 97 pgs: 97 active+clean; 350 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 4.6 MiB/s wr, 404 op/s; 318 KiB/s, 42 objects/s recovering 2024-09-21T10:46:31.839 INFO:teuthology.orchestra.run.smithi123.stdout: 5 227484 9.96 MB/sec execute 452 sec latency 51.588 ms 2024-09-21T10:46:32.839 INFO:teuthology.orchestra.run.smithi123.stdout: 5 227699 9.96 MB/sec execute 453 sec latency 83.764 ms 2024-09-21T10:46:33.571 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:33 smithi123 ceph-mon[63856]: pgmap v204: 97 pgs: 97 active+clean; 353 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3.7 KiB/s rd, 3.3 MiB/s wr, 322 op/s; 280 KiB/s, 21 objects/s recovering 2024-09-21T10:46:33.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:33 smithi175 ceph-mon[49304]: pgmap v204: 97 pgs: 97 active+clean; 353 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3.7 KiB/s rd, 3.3 MiB/s wr, 322 op/s; 280 KiB/s, 21 objects/s recovering 2024-09-21T10:46:33.839 INFO:teuthology.orchestra.run.smithi123.stdout: 5 227917 9.95 MB/sec execute 454 sec latency 69.803 ms 2024-09-21T10:46:34.596 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:34 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-21T10:46:34.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:34 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-21T10:46:34.839 INFO:teuthology.orchestra.run.smithi123.stdout: 5 228234 9.94 MB/sec execute 455 sec latency 116.490 ms 2024-09-21T10:46:35.605 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:35 smithi123 ceph-mon[63856]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-21T10:46:35.605 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:35 smithi123 ceph-mon[63856]: Upgrade: osd.4 is safe to restart 2024-09-21T10:46:35.605 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:35 smithi123 ceph-mon[63856]: pgmap v205: 97 pgs: 97 active+clean; 350 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.5 MiB/s wr, 440 op/s; 236 KiB/s, 17 objects/s recovering 2024-09-21T10:46:35.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:35 smithi175 ceph-mon[49304]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-21T10:46:35.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:35 smithi175 ceph-mon[49304]: Upgrade: osd.4 is safe to restart 2024-09-21T10:46:35.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:35 smithi175 ceph-mon[49304]: pgmap v205: 97 pgs: 97 active+clean; 350 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.5 MiB/s wr, 440 op/s; 236 KiB/s, 17 objects/s recovering 2024-09-21T10:46:35.839 INFO:teuthology.orchestra.run.smithi123.stdout: 5 228653 9.93 MB/sec execute 456 sec latency 43.641 ms 2024-09-21T10:46:36.839 INFO:teuthology.orchestra.run.smithi123.stdout: 5 228957 9.92 MB/sec execute 457 sec latency 46.232 ms 2024-09-21T10:46:36.897 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:36 smithi123 ceph-mon[63856]: Upgrade: Updating osd.4 2024-09-21T10:46:36.897 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:36 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:46:36.897 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:36 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-21T10:46:36.897 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:36 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:46:36.897 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:36 smithi123 ceph-mon[63856]: Deploying daemon osd.4 on smithi175 2024-09-21T10:46:37.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:36 smithi175 ceph-mon[49304]: Upgrade: Updating osd.4 2024-09-21T10:46:37.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:36 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:46:37.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:36 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-21T10:46:37.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:36 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:46:37.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:36 smithi175 ceph-mon[49304]: Deploying daemon osd.4 on smithi175 2024-09-21T10:46:37.839 INFO:teuthology.orchestra.run.smithi123.stdout: 5 229291 9.92 MB/sec execute 458 sec latency 41.683 ms 2024-09-21T10:46:37.884 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:37 smithi123 ceph-mon[63856]: pgmap v206: 97 pgs: 97 active+clean; 349 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.3 MiB/s wr, 399 op/s; 82 KiB/s, 11 objects/s recovering 2024-09-21T10:46:37.884 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:37 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:46:38.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:37 smithi175 ceph-mon[49304]: pgmap v206: 97 pgs: 97 active+clean; 349 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.3 MiB/s wr, 399 op/s; 82 KiB/s, 11 objects/s recovering 2024-09-21T10:46:38.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:37 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:46:38.839 INFO:teuthology.orchestra.run.smithi123.stdout: 5 229743 9.93 MB/sec execute 459 sec latency 44.842 ms 2024-09-21T10:46:39.124 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:46:38 smithi175 systemd[1]: Stopping Ceph osd.4 for 3ccd15e6-7804-11ef-baf6-efdc52797490... 2024-09-21T10:46:39.623 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:46:39 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-4[29923]: 2024-09-21T10:46:39.126+0000 7fd841c63700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:46:39.623 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:46:39 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-4[29923]: 2024-09-21T10:46:39.126+0000 7fd841c63700 -1 osd.4 132 *** Got signal Terminated *** 2024-09-21T10:46:39.623 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:46:39 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-4[29923]: 2024-09-21T10:46:39.126+0000 7fd841c63700 -1 osd.4 132 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T10:46:39.839 INFO:teuthology.orchestra.run.smithi123.stdout: 5 230093 9.93 MB/sec execute 460 sec latency 69.838 ms 2024-09-21T10:46:40.100 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:39 smithi123 ceph-mon[63856]: pgmap v207: 97 pgs: 97 active+clean; 354 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.9 MiB/s wr, 524 op/s 2024-09-21T10:46:40.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:39 smithi123 ceph-mon[63856]: osd.4 marked itself down and dead 2024-09-21T10:46:40.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:39 smithi175 ceph-mon[49304]: pgmap v207: 97 pgs: 97 active+clean; 354 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.9 MiB/s wr, 524 op/s 2024-09-21T10:46:40.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:39 smithi175 ceph-mon[49304]: osd.4 marked itself down and dead 2024-09-21T10:46:40.571 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:46:40 smithi175 podman[52735]: 2024-09-21 10:46:40.291553491 +0000 UTC m=+1.297896870 container died 48fe728c259e0c4baa64ce62b6852ed73fc82bc3916226b3cd32fa3d13377c9d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-4, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, version=8, name=centos-stream, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, distribution-scope=public, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-09-21T10:46:40.840 INFO:teuthology.orchestra.run.smithi123.stdout: 5 230123 9.92 MB/sec execute 461 sec latency 1022.135 ms 2024-09-21T10:46:40.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:40 smithi175 ceph-mon[49304]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T10:46:40.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:40 smithi175 ceph-mon[49304]: osdmap e133: 8 total, 7 up, 8 in 2024-09-21T10:46:40.873 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:46:40 smithi175 podman[52735]: 2024-09-21 10:46:40.669133026 +0000 UTC m=+1.675476399 container cleanup 48fe728c259e0c4baa64ce62b6852ed73fc82bc3916226b3cd32fa3d13377c9d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-4, architecture=x86_64, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, com.redhat.component=centos-stream-container, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, version=8, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-09-21T10:46:40.874 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:46:40 smithi175 bash[52735]: ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-4 2024-09-21T10:46:41.003 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:40 smithi123 ceph-mon[63856]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T10:46:41.003 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:40 smithi123 ceph-mon[63856]: osdmap e133: 8 total, 7 up, 8 in 2024-09-21T10:46:41.356 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:46:40 smithi175 podman[52751]: 2024-09-21 10:46:40.954046974 +0000 UTC m=+0.656905850 container remove 48fe728c259e0c4baa64ce62b6852ed73fc82bc3916226b3cd32fa3d13377c9d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-4, vcs-type=git, release=754, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, architecture=x86_64, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, ceph=True, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , vendor=Red Hat, Inc.) 2024-09-21T10:46:41.623 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:46:41 smithi175 podman[52838]: 2024-09-21 10:46:41.258199214 +0000 UTC m=+0.018339732 image pull 8f4d013db793795fef09ff5edf96e6b42ba8cd2c9158b2f0b7a5d7c1e657c063 quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472 2024-09-21T10:46:41.623 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:46:41 smithi175 podman[52838]: 2024-09-21 10:46:41.363299902 +0000 UTC m=+0.123440407 container create 4afe2216c2e913024fbe226208482a5b11aab7b43e17d9bd877f0d600ed92308 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-4-deactivate, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T10:46:41.772 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:41 smithi123 ceph-mon[63856]: pgmap v209: 97 pgs: 8 peering, 12 stale+active+clean, 77 active+clean; 350 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.9 MiB/s wr, 566 op/s 2024-09-21T10:46:41.772 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:41 smithi123 ceph-mon[63856]: osdmap e134: 8 total, 7 up, 8 in 2024-09-21T10:46:41.840 INFO:teuthology.orchestra.run.smithi123.stdout: 5 230389 9.92 MB/sec execute 462 sec latency 59.877 ms 2024-09-21T10:46:42.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:41 smithi175 ceph-mon[49304]: pgmap v209: 97 pgs: 8 peering, 12 stale+active+clean, 77 active+clean; 350 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.9 MiB/s wr, 566 op/s 2024-09-21T10:46:42.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:41 smithi175 ceph-mon[49304]: osdmap e134: 8 total, 7 up, 8 in 2024-09-21T10:46:42.509 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:46:42 smithi175 podman[52838]: 2024-09-21 10:46:42.283923324 +0000 UTC m=+1.044063840 container init 4afe2216c2e913024fbe226208482a5b11aab7b43e17d9bd877f0d600ed92308 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-4-deactivate, org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-09-21T10:46:42.509 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:46:42 smithi175 podman[52838]: 2024-09-21 10:46:42.288528621 +0000 UTC m=+1.048669135 container start 4afe2216c2e913024fbe226208482a5b11aab7b43e17d9bd877f0d600ed92308 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-4-deactivate, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240731) 2024-09-21T10:46:42.509 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:46:42 smithi175 podman[52838]: 2024-09-21 10:46:42.392227187 +0000 UTC m=+1.152367700 container attach 4afe2216c2e913024fbe226208482a5b11aab7b43e17d9bd877f0d600ed92308 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-4-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-09-21T10:46:42.839 INFO:teuthology.orchestra.run.smithi123.stdout: 5 230627 9.92 MB/sec execute 463 sec latency 48.621 ms 2024-09-21T10:46:42.873 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:46:42 smithi175 podman[52838]: 2024-09-21 10:46:42.507859626 +0000 UTC m=+1.268000147 container died 4afe2216c2e913024fbe226208482a5b11aab7b43e17d9bd877f0d600ed92308 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-4-deactivate, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T10:46:43.671 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:43 smithi175 ceph-mon[49304]: pgmap v211: 97 pgs: 13 peering, 9 stale+active+clean, 75 active+clean; 351 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 3.6 MiB/s wr, 351 op/s 2024-09-21T10:46:43.672 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:43 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:46:43.672 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:43 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:46:43.672 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:46:43 smithi175 podman[52860]: 2024-09-21 10:46:43.423010248 +0000 UTC m=+0.908022067 container remove 4afe2216c2e913024fbe226208482a5b11aab7b43e17d9bd877f0d600ed92308 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-4-deactivate, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=) 2024-09-21T10:46:43.672 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:46:43 smithi175 systemd[1]: ceph-3ccd15e6-7804-11ef-baf6-efdc52797490@osd.4.service: Deactivated successfully. 2024-09-21T10:46:43.672 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:46:43 smithi175 systemd[1]: Stopped Ceph osd.4 for 3ccd15e6-7804-11ef-baf6-efdc52797490. 2024-09-21T10:46:43.672 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:46:43 smithi175 systemd[1]: ceph-3ccd15e6-7804-11ef-baf6-efdc52797490@osd.4.service: Consumed 4min 9.378s CPU time. 2024-09-21T10:46:43.672 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:46:43 smithi175 systemd[1]: Starting Ceph osd.4 for 3ccd15e6-7804-11ef-baf6-efdc52797490... 2024-09-21T10:46:43.801 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:43 smithi123 ceph-mon[63856]: pgmap v211: 97 pgs: 13 peering, 9 stale+active+clean, 75 active+clean; 351 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 3.6 MiB/s wr, 351 op/s 2024-09-21T10:46:43.801 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:43 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:46:43.801 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:43 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:46:43.840 INFO:teuthology.orchestra.run.smithi123.stdout: 5 230866 9.91 MB/sec execute 464 sec latency 72.157 ms 2024-09-21T10:46:43.926 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:46:43 smithi175 podman[52952]: 2024-09-21 10:46:43.798171496 +0000 UTC m=+0.019973905 image pull 8f4d013db793795fef09ff5edf96e6b42ba8cd2c9158b2f0b7a5d7c1e657c063 quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472 2024-09-21T10:46:43.926 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:46:43 smithi175 podman[52952]: 2024-09-21 10:46:43.924445661 +0000 UTC m=+0.146248063 container create e79937a12a248123e731afcf1bbe5328ed5a7d6c9da6ca26c1b1ec751b0ebc40 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-4-activate, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, org.label-schema.license=GPLv2) 2024-09-21T10:46:44.574 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:46:44 smithi175 podman[52952]: 2024-09-21 10:46:44.185692072 +0000 UTC m=+0.407494471 container init e79937a12a248123e731afcf1bbe5328ed5a7d6c9da6ca26c1b1ec751b0ebc40 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-4-activate, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-09-21T10:46:44.575 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:46:44 smithi175 podman[52952]: 2024-09-21 10:46:44.190268395 +0000 UTC m=+0.412070798 container start e79937a12a248123e731afcf1bbe5328ed5a7d6c9da6ca26c1b1ec751b0ebc40 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-4-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, ceph=True, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2) 2024-09-21T10:46:44.575 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:46:44 smithi175 podman[52952]: 2024-09-21 10:46:44.285870066 +0000 UTC m=+0.507672483 container attach e79937a12a248123e731afcf1bbe5328ed5a7d6c9da6ca26c1b1ec751b0ebc40 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-4-activate, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731) 2024-09-21T10:46:44.575 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:46:44 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-4-activate[52967]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:46:44.575 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:46:44 smithi175 bash[52952]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:46:44.575 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:46:44 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-4-activate[52967]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:46:44.575 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:46:44 smithi175 bash[52952]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:46:44.813 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:44 smithi123 ceph-mon[63856]: Health check failed: Degraded data redundancy: 174/1674 objects degraded (10.394%), 20 pgs degraded (PG_DEGRADED) 2024-09-21T10:46:44.840 INFO:teuthology.orchestra.run.smithi123.stdout: 5 231139 9.90 MB/sec execute 465 sec latency 49.116 ms 2024-09-21T10:46:44.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:44 smithi175 ceph-mon[49304]: Health check failed: Degraded data redundancy: 174/1674 objects degraded (10.394%), 20 pgs degraded (PG_DEGRADED) 2024-09-21T10:46:45.825 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:45 smithi123 ceph-mon[63856]: pgmap v212: 97 pgs: 8 active+undersized, 13 peering, 20 active+undersized+degraded, 56 active+clean; 356 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.9 MiB/s wr, 509 op/s; 174/1674 objects degraded (10.394%) 2024-09-21T10:46:45.839 INFO:teuthology.orchestra.run.smithi123.stdout: 5 231355 9.89 MB/sec execute 466 sec latency 57.349 ms 2024-09-21T10:46:46.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:45 smithi175 ceph-mon[49304]: pgmap v212: 97 pgs: 8 active+undersized, 13 peering, 20 active+undersized+degraded, 56 active+clean; 356 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.9 MiB/s wr, 509 op/s; 174/1674 objects degraded (10.394%) 2024-09-21T10:46:46.840 INFO:teuthology.orchestra.run.smithi123.stdout: 5 231643 9.88 MB/sec execute 467 sec latency 44.223 ms 2024-09-21T10:46:47.624 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:46:47 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-4-activate[52967]: --> Failed to activate via raw: did not find any matching OSD to activate 2024-09-21T10:46:47.624 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:46:47 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-4-activate[52967]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:46:47.624 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:46:47 smithi175 bash[52952]: --> Failed to activate via raw: did not find any matching OSD to activate 2024-09-21T10:46:47.624 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:46:47 smithi175 bash[52952]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:46:47.624 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:46:47 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-4-activate[52967]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:46:47.624 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:46:47 smithi175 bash[52952]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:46:47.624 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:46:47 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-4-activate[52967]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-21T10:46:47.624 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:46:47 smithi175 bash[52952]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-21T10:46:47.624 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:46:47 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-4-activate[52967]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-4 --no-mon-config 2024-09-21T10:46:47.625 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:46:47 smithi175 bash[52952]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-4 --no-mon-config 2024-09-21T10:46:47.840 INFO:teuthology.orchestra.run.smithi123.stdout: 5 232026 9.87 MB/sec execute 468 sec latency 55.162 ms 2024-09-21T10:46:47.868 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:47 smithi123 ceph-mon[63856]: pgmap v213: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 357 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.2 MiB/s wr, 306 op/s; 258/1689 objects degraded (15.275%) 2024-09-21T10:46:48.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:47 smithi175 ceph-mon[49304]: pgmap v213: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 357 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.2 MiB/s wr, 306 op/s; 258/1689 objects degraded (15.275%) 2024-09-21T10:46:48.124 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:46:47 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-4-activate[52967]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-4/block 2024-09-21T10:46:48.124 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:46:47 smithi175 bash[52952]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-4/block 2024-09-21T10:46:48.124 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:46:47 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-4-activate[52967]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-4/block 2024-09-21T10:46:48.124 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:46:47 smithi175 bash[52952]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-4/block 2024-09-21T10:46:48.124 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:46:47 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-4-activate[52967]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-21T10:46:48.124 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:46:47 smithi175 bash[52952]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-21T10:46:48.124 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:46:47 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-4-activate[52967]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-21T10:46:48.124 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:46:47 smithi175 bash[52952]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-21T10:46:48.124 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:46:47 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-4-activate[52967]: --> ceph-volume lvm activate successful for osd ID: 4 2024-09-21T10:46:48.125 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:46:47 smithi175 bash[52952]: --> ceph-volume lvm activate successful for osd ID: 4 2024-09-21T10:46:48.125 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:46:47 smithi175 podman[52952]: 2024-09-21 10:46:47.705901511 +0000 UTC m=+3.927703923 container died e79937a12a248123e731afcf1bbe5328ed5a7d6c9da6ca26c1b1ec751b0ebc40 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-4-activate, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-09-21T10:46:48.836 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:46:48 smithi175 podman[53201]: 2024-09-21 10:46:48.583862166 +0000 UTC m=+0.870846889 container remove e79937a12a248123e731afcf1bbe5328ed5a7d6c9da6ca26c1b1ec751b0ebc40 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-4-activate, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS) 2024-09-21T10:46:48.836 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:46:48 smithi175 podman[53255]: 2024-09-21 10:46:48.710804989 +0000 UTC m=+0.019045562 image pull 8f4d013db793795fef09ff5edf96e6b42ba8cd2c9158b2f0b7a5d7c1e657c063 quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472 2024-09-21T10:46:48.839 INFO:teuthology.orchestra.run.smithi123.stdout: 5 232411 9.86 MB/sec execute 469 sec latency 38.834 ms 2024-09-21T10:46:49.109 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:46:48 smithi175 podman[53255]: 2024-09-21 10:46:48.83482642 +0000 UTC m=+0.143066984 container create b650f72d10f046eaada2771b4eb808f48067cdcc25727c8d706f941cb70e0bea (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-4, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, ceph=True, CEPH_POINT_RELEASE=) 2024-09-21T10:46:49.110 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:46:49 smithi175 podman[53255]: 2024-09-21 10:46:49.079761227 +0000 UTC m=+0.388001799 container init b650f72d10f046eaada2771b4eb808f48067cdcc25727c8d706f941cb70e0bea (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-4, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-09-21T10:46:49.110 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:46:49 smithi175 podman[53255]: 2024-09-21 10:46:49.083145632 +0000 UTC m=+0.391386203 container start b650f72d10f046eaada2771b4eb808f48067cdcc25727c8d706f941cb70e0bea (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-4, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS) 2024-09-21T10:46:49.373 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:46:49 smithi175 ceph-osd[53273]: -- 172.21.15.175:0/1582169961 <== mon.1 v2:172.21.15.175:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55c2f1cf81e0 con 0x55c2f0e97c00 2024-09-21T10:46:49.373 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:46:49 smithi175 bash[53255]: b650f72d10f046eaada2771b4eb808f48067cdcc25727c8d706f941cb70e0bea 2024-09-21T10:46:49.373 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:46:49 smithi175 systemd[1]: Started Ceph osd.4 for 3ccd15e6-7804-11ef-baf6-efdc52797490. 2024-09-21T10:46:49.840 INFO:teuthology.orchestra.run.smithi123.stdout: 5 232692 9.87 MB/sec execute 470 sec latency 52.504 ms 2024-09-21T10:46:49.945 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:49 smithi123 ceph-mon[63856]: pgmap v214: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 356 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 2.7 KiB/s rd, 3.5 MiB/s wr, 263 op/s; 263/1713 objects degraded (15.353%) 2024-09-21T10:46:49.945 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:49 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:46:49.945 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:49 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:46:49.945 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:49 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:46:50.054 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:49 smithi175 ceph-mon[49304]: pgmap v214: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 356 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 2.7 KiB/s rd, 3.5 MiB/s wr, 263 op/s; 263/1713 objects degraded (15.353%) 2024-09-21T10:46:50.055 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:49 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:46:50.055 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:49 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:46:50.055 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:49 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:46:50.702 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:50 smithi123 ceph-mon[63856]: Health check update: Degraded data redundancy: 263/1713 objects degraded (15.353%), 28 pgs degraded (PG_DEGRADED) 2024-09-21T10:46:50.840 INFO:teuthology.orchestra.run.smithi123.stdout: 5 233066 9.87 MB/sec execute 471 sec latency 52.865 ms 2024-09-21T10:46:50.974 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:50 smithi175 ceph-mon[49304]: Health check update: Degraded data redundancy: 263/1713 objects degraded (15.353%), 28 pgs degraded (PG_DEGRADED) 2024-09-21T10:46:51.373 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:46:50 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-4[53269]: 2024-09-21T10:46:50.969+0000 7fb517d17640 -1 Falling back to public interface 2024-09-21T10:46:51.779 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:51 smithi123 ceph-mon[63856]: pgmap v215: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 355 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.1 MiB/s wr, 401 op/s; 263/1665 objects degraded (15.796%) 2024-09-21T10:46:51.840 INFO:teuthology.orchestra.run.smithi123.stdout: 5 233454 9.87 MB/sec execute 472 sec latency 96.686 ms 2024-09-21T10:46:51.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:51 smithi175 ceph-mon[49304]: pgmap v215: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 355 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.1 MiB/s wr, 401 op/s; 263/1665 objects degraded (15.796%) 2024-09-21T10:46:52.840 INFO:teuthology.orchestra.run.smithi123.stdout: 5 233812 9.88 MB/sec execute 473 sec latency 54.935 ms 2024-09-21T10:46:53.822 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:53 smithi123 ceph-mon[63856]: pgmap v216: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 352 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.4 MiB/s wr, 344 op/s; 261/1644 objects degraded (15.876%) 2024-09-21T10:46:53.822 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:53 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:46:53.822 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:53 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:46:53.822 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:53 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:46:53.822 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:53 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:46:53.840 INFO:teuthology.orchestra.run.smithi123.stdout: 5 234127 9.88 MB/sec execute 474 sec latency 43.445 ms 2024-09-21T10:46:54.003 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:53 smithi175 ceph-mon[49304]: pgmap v216: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 352 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.4 MiB/s wr, 344 op/s; 261/1644 objects degraded (15.876%) 2024-09-21T10:46:54.003 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:53 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:46:54.003 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:53 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:46:54.004 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:53 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:46:54.004 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:53 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:46:54.119 INFO:teuthology.orchestra.run.smithi123.stdout:true 2024-09-21T10:46:54.840 INFO:teuthology.orchestra.run.smithi123.stdout: 5 234520 9.88 MB/sec execute 475 sec latency 71.715 ms 2024-09-21T10:46:55.282 INFO:teuthology.orchestra.run.smithi123.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T10:46:55.282 INFO:teuthology.orchestra.run.smithi123.stdout:alertmanager.smithi123 smithi123 *:9093,9094 running (7m) 43s ago 16m 22.0M - 0.27.0 11f11916f8cd 5d550bd97de5 2024-09-21T10:46:55.282 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi123 smithi123 running (4m) 43s ago 16m 6899k - 19.3.0-5103-g5c1cdc3c 8f4d013db793 c5e1ef39a0a8 2024-09-21T10:46:55.282 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi175 smithi175 running (4m) 2s ago 15m 6895k - 19.3.0-5103-g5c1cdc3c 8f4d013db793 fb1ca999ccfd 2024-09-21T10:46:55.283 INFO:teuthology.orchestra.run.smithi123.stdout:grafana.smithi123 smithi123 *:3000 running (7m) 43s ago 16m 52.6M - dad864ee21e9 771df81763ba 2024-09-21T10:46:55.283 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi123.uqoyow smithi123 running (10m) 43s ago 10m 41.2M - 17.2.0 e1d6a67b021e f0a21eaac266 2024-09-21T10:46:55.283 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi175.jrlevw smithi175 running (10m) 2s ago 10m 14.3M - 17.2.0 e1d6a67b021e b21b16f4bc50 2024-09-21T10:46:55.283 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi123.stnuit smithi123 *:8443,9283,8765 running (6m) 43s ago 17m 519M - 19.3.0-5103-g5c1cdc3c 8f4d013db793 6812cc5cb56b 2024-09-21T10:46:55.283 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi175.txvwrq smithi175 *:8443,9283,8765 running (5m) 2s ago 15m 456M - 19.3.0-5103-g5c1cdc3c 8f4d013db793 304f0408075e 2024-09-21T10:46:55.283 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi123 smithi123 running (5m) 43s ago 17m 52.0M 2048M 19.3.0-5103-g5c1cdc3c 8f4d013db793 6b95207a2d80 2024-09-21T10:46:55.283 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi175 smithi175 running (4m) 2s ago 15m 42.0M 2048M 19.3.0-5103-g5c1cdc3c 8f4d013db793 3abee2d37f0f 2024-09-21T10:46:55.283 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.0.0.smithi123.ddatpr smithi123 *:2049 running (10m) 43s ago 10m 281M - 4.0 e1d6a67b021e 36f8cb2281eb 2024-09-21T10:46:55.283 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.1.0.smithi175.wpawer smithi175 *:2049 running (10m) 2s ago 10m 90.7M - 4.0 e1d6a67b021e 2200aeed9200 2024-09-21T10:46:55.283 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi123 smithi123 *:9100 running (7m) 43s ago 16m 12.3M - 1.7.0 72c9c2088986 ca7d0b29af81 2024-09-21T10:46:55.283 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi175 smithi175 *:9100 running (6m) 2s ago 15m 10.6M - 1.7.0 72c9c2088986 46c4f6b7c149 2024-09-21T10:46:55.283 INFO:teuthology.orchestra.run.smithi123.stdout:osd.0 smithi123 running (3m) 43s ago 14m 362M 1936M 19.3.0-5103-g5c1cdc3c 8f4d013db793 cc40f695c3a8 2024-09-21T10:46:55.283 INFO:teuthology.orchestra.run.smithi123.stdout:osd.1 smithi123 running (2m) 43s ago 14m 279M 1936M 19.3.0-5103-g5c1cdc3c 8f4d013db793 1be249025c7f 2024-09-21T10:46:55.284 INFO:teuthology.orchestra.run.smithi123.stdout:osd.2 smithi123 running (102s) 43s ago 13m 232M 1936M 19.3.0-5103-g5c1cdc3c 8f4d013db793 647aec759e3c 2024-09-21T10:46:55.284 INFO:teuthology.orchestra.run.smithi123.stdout:osd.3 smithi123 running (52s) 43s ago 13m 11.6M 1936M 19.3.0-5103-g5c1cdc3c 8f4d013db793 7a082fcad8e8 2024-09-21T10:46:55.284 INFO:teuthology.orchestra.run.smithi123.stdout:osd.4 smithi175 running (6s) 2s ago 13m 11.5M 2704M 19.3.0-5103-g5c1cdc3c 8f4d013db793 b650f72d10f0 2024-09-21T10:46:55.284 INFO:teuthology.orchestra.run.smithi123.stdout:osd.5 smithi175 running (12m) 2s ago 12m 561M 2704M 17.2.0 e1d6a67b021e e270d5a21b14 2024-09-21T10:46:55.284 INFO:teuthology.orchestra.run.smithi123.stdout:osd.6 smithi175 running (12m) 2s ago 12m 488M 2704M 17.2.0 e1d6a67b021e 9bcbf018eb20 2024-09-21T10:46:55.284 INFO:teuthology.orchestra.run.smithi123.stdout:osd.7 smithi175 running (11m) 2s ago 11m 707M 2704M 17.2.0 e1d6a67b021e e9b8f1f90994 2024-09-21T10:46:55.284 INFO:teuthology.orchestra.run.smithi123.stdout:prometheus.smithi123 smithi123 *:9095 running (5m) 43s ago 15m 55.4M - 2.51.0 1d3b7f56885b 37f944c9de88 2024-09-21T10:46:55.840 INFO:teuthology.orchestra.run.smithi123.stdout: 5 234775 9.87 MB/sec execute 476 sec latency 39.214 ms 2024-09-21T10:46:55.849 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:55 smithi123 ceph-mon[63856]: from='client.34358 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:46:55.849 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:55 smithi123 ceph-mon[63856]: pgmap v217: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 349 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.7 MiB/s wr, 508 op/s; 261/1644 objects degraded (15.876%) 2024-09-21T10:46:55.849 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:55 smithi123 ceph-mon[63856]: from='client.44305 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:46:55.849 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:55 smithi123 ceph-mon[63856]: Health check update: Degraded data redundancy: 261/1644 objects degraded (15.876%), 28 pgs degraded (PG_DEGRADED) 2024-09-21T10:46:55.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:55 smithi175 ceph-mon[49304]: from='client.34358 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:46:55.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:55 smithi175 ceph-mon[49304]: pgmap v217: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 349 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.7 MiB/s wr, 508 op/s; 261/1644 objects degraded (15.876%) 2024-09-21T10:46:55.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:55 smithi175 ceph-mon[49304]: from='client.44305 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:46:55.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:55 smithi175 ceph-mon[49304]: Health check update: Degraded data redundancy: 261/1644 objects degraded (15.876%), 28 pgs degraded (PG_DEGRADED) 2024-09-21T10:46:55.924 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-09-21T10:46:55.924 INFO:teuthology.orchestra.run.smithi123.stdout: "mon": { 2024-09-21T10:46:55.924 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev)": 2 2024-09-21T10:46:55.924 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:46:55.924 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr": { 2024-09-21T10:46:55.924 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev)": 2 2024-09-21T10:46:55.924 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:46:55.924 INFO:teuthology.orchestra.run.smithi123.stdout: "osd": { 2024-09-21T10:46:55.924 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-09-21T10:46:55.924 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev)": 4 2024-09-21T10:46:55.924 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:46:55.925 INFO:teuthology.orchestra.run.smithi123.stdout: "mds": { 2024-09-21T10:46:55.925 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T10:46:55.925 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:46:55.925 INFO:teuthology.orchestra.run.smithi123.stdout: "overall": { 2024-09-21T10:46:55.925 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-09-21T10:46:55.925 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev)": 8 2024-09-21T10:46:55.925 INFO:teuthology.orchestra.run.smithi123.stdout: } 2024-09-21T10:46:55.925 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-09-21T10:46:56.476 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-09-21T10:46:56.476 INFO:teuthology.orchestra.run.smithi123.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472", 2024-09-21T10:46:56.476 INFO:teuthology.orchestra.run.smithi123.stdout: "in_progress": true, 2024-09-21T10:46:56.476 INFO:teuthology.orchestra.run.smithi123.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T10:46:56.476 INFO:teuthology.orchestra.run.smithi123.stdout: "services_complete": [ 2024-09-21T10:46:56.476 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr", 2024-09-21T10:46:56.476 INFO:teuthology.orchestra.run.smithi123.stdout: "crash", 2024-09-21T10:46:56.476 INFO:teuthology.orchestra.run.smithi123.stdout: "mon" 2024-09-21T10:46:56.476 INFO:teuthology.orchestra.run.smithi123.stdout: ], 2024-09-21T10:46:56.476 INFO:teuthology.orchestra.run.smithi123.stdout: "progress": "11/23 daemons upgraded", 2024-09-21T10:46:56.477 INFO:teuthology.orchestra.run.smithi123.stdout: "message": "Currently upgrading osd daemons", 2024-09-21T10:46:56.477 INFO:teuthology.orchestra.run.smithi123.stdout: "is_paused": false 2024-09-21T10:46:56.477 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-09-21T10:46:56.840 INFO:teuthology.orchestra.run.smithi123.stdout: 5 234953 9.86 MB/sec execute 477 sec latency 60.582 ms 2024-09-21T10:46:56.880 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:56 smithi123 ceph-mon[63856]: from='client.34366 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:46:56.880 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:56 smithi123 ceph-mon[63856]: from='client.? 172.21.15.123:0/644129079' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:46:57.043 INFO:teuthology.orchestra.run.smithi123.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 264/1698 objects degraded (15.548%), 28 pgs degraded 2024-09-21T10:46:57.043 INFO:teuthology.orchestra.run.smithi123.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-21T10:46:57.043 INFO:teuthology.orchestra.run.smithi123.stdout: osd.4 (root=default,host=smithi175) is down 2024-09-21T10:46:57.043 INFO:teuthology.orchestra.run.smithi123.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 264/1698 objects degraded (15.548%), 28 pgs degraded 2024-09-21T10:46:57.044 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.5 is active+undersized+degraded, acting [7,0] 2024-09-21T10:46:57.044 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.6 is active+undersized+degraded, acting [1,6] 2024-09-21T10:46:57.044 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.b is active+undersized+degraded, acting [7,5] 2024-09-21T10:46:57.044 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.d is active+undersized+degraded, acting [1,3] 2024-09-21T10:46:57.044 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.f is active+undersized+degraded, acting [0,7] 2024-09-21T10:46:57.045 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.11 is active+undersized+degraded, acting [6,1] 2024-09-21T10:46:57.045 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.13 is active+undersized+degraded, acting [0,3] 2024-09-21T10:46:57.045 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.18 is active+undersized+degraded, acting [5,7] 2024-09-21T10:46:57.045 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.1a is active+undersized+degraded, acting [6,7] 2024-09-21T10:46:57.045 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.1f is active+undersized+degraded, acting [0,7] 2024-09-21T10:46:57.045 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.1 is active+undersized+degraded, acting [0,3] 2024-09-21T10:46:57.045 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.3 is active+undersized+degraded, acting [0,6] 2024-09-21T10:46:57.046 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.6 is active+undersized+degraded, acting [0,1] 2024-09-21T10:46:57.046 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.9 is active+undersized+degraded, acting [2,7] 2024-09-21T10:46:57.046 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.a is active+undersized+degraded, acting [6,1] 2024-09-21T10:46:57.046 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.b is active+undersized+degraded, acting [3,0] 2024-09-21T10:46:57.046 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.e is active+undersized+degraded, acting [7,1] 2024-09-21T10:46:57.046 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.f is active+undersized+degraded, acting [7,0] 2024-09-21T10:46:57.047 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.11 is active+undersized+degraded, acting [7,6] 2024-09-21T10:46:57.047 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.13 is active+undersized+degraded, acting [7,2] 2024-09-21T10:46:57.047 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.14 is active+undersized+degraded, acting [7,6] 2024-09-21T10:46:57.047 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.15 is active+undersized+degraded, acting [7,3] 2024-09-21T10:46:57.047 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.19 is active+undersized+degraded, acting [1,3] 2024-09-21T10:46:57.047 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.1a is active+undersized+degraded, acting [1,2] 2024-09-21T10:46:57.048 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.1b is active+undersized+degraded, acting [0,7] 2024-09-21T10:46:57.048 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.1c is active+undersized+degraded, acting [5,1] 2024-09-21T10:46:57.048 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.1d is active+undersized+degraded, acting [5,6] 2024-09-21T10:46:57.048 INFO:teuthology.orchestra.run.smithi123.stdout: pg 4.5 is active+undersized+degraded, acting [6,7] 2024-09-21T10:46:57.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:56 smithi175 ceph-mon[49304]: from='client.34366 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:46:57.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:56 smithi175 ceph-mon[49304]: from='client.? 172.21.15.123:0/644129079' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:46:57.840 INFO:teuthology.orchestra.run.smithi123.stdout: 5 235231 9.86 MB/sec execute 478 sec latency 60.827 ms 2024-09-21T10:46:57.892 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:57 smithi123 ceph-mon[63856]: pgmap v218: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 355 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.3 MiB/s wr, 414 op/s; 264/1698 objects degraded (15.548%) 2024-09-21T10:46:57.893 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:57 smithi123 ceph-mon[63856]: from='client.44317 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:46:57.893 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:57 smithi123 ceph-mon[63856]: from='client.? 172.21.15.123:0/3373295569' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T10:46:57.893 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:57 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:46:57.893 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:57 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:46:57.893 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:57 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:46:57.893 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:57 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:46:57.893 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:57 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:46:57.893 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:57 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:46:57.893 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:57 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:46:57.893 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:57 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:46:57.893 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:57 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:46:57.894 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:57 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:46:57.894 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:57 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:46:57.894 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:57 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:46:57.894 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:57 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-21T10:46:58.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:57 smithi175 ceph-mon[49304]: pgmap v218: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 355 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.3 MiB/s wr, 414 op/s; 264/1698 objects degraded (15.548%) 2024-09-21T10:46:58.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:57 smithi175 ceph-mon[49304]: from='client.44317 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:46:58.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:57 smithi175 ceph-mon[49304]: from='client.? 172.21.15.123:0/3373295569' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T10:46:58.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:57 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:46:58.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:57 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:46:58.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:57 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:46:58.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:57 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:46:58.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:57 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:46:58.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:57 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:46:58.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:57 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:46:58.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:57 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:46:58.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:57 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:46:58.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:57 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:46:58.125 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:57 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:46:58.125 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:57 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:46:58.125 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:57 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-21T10:46:58.705 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:46:58 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-4[53269]: 2024-09-21T10:46:58.415+0000 7fb517d17640 -1 osd.4 0 read_superblock omap replica is missing. 2024-09-21T10:46:58.769 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:58 smithi123 ceph-mon[63856]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-21T10:46:58.769 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:58 smithi123 ceph-mon[63856]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-09-21T10:46:58.840 INFO:teuthology.orchestra.run.smithi123.stdout: 5 235568 9.85 MB/sec execute 479 sec latency 59.053 ms 2024-09-21T10:46:59.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:58 smithi175 ceph-mon[49304]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-21T10:46:59.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:58 smithi175 ceph-mon[49304]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-09-21T10:46:59.792 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:46:59 smithi123 ceph-mon[63856]: pgmap v219: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 359 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.6 MiB/s wr, 490 op/s; 260/1689 objects degraded (15.394%) 2024-09-21T10:46:59.840 INFO:teuthology.orchestra.run.smithi123.stdout: 5 235892 9.84 MB/sec execute 480 sec latency 47.090 ms 2024-09-21T10:47:00.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:46:59 smithi175 ceph-mon[49304]: pgmap v219: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 359 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.6 MiB/s wr, 490 op/s; 260/1689 objects degraded (15.394%) 2024-09-21T10:47:00.123 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:46:59 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-4[53269]: 2024-09-21T10:46:59.715+0000 7fb517d17640 -1 osd.4 132 log_to_monitors true 2024-09-21T10:47:00.815 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:00 smithi123 ceph-mon[63856]: from='osd.4 [v2:172.21.15.175:6800/2327534492,v1:172.21.15.175:6801/2327534492]' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-21T10:47:00.815 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:00 smithi123 ceph-mon[63856]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-21T10:47:00.815 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:00 smithi123 ceph-mon[63856]: Health check update: Degraded data redundancy: 260/1689 objects degraded (15.394%), 28 pgs degraded (PG_DEGRADED) 2024-09-21T10:47:00.840 INFO:teuthology.orchestra.run.smithi123.stdout: 5 236130 9.83 MB/sec execute 481 sec latency 65.485 ms 2024-09-21T10:47:01.123 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:47:00 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-4[53269]: 2024-09-21T10:47:00.731+0000 7fb50fac1640 -1 osd.4 132 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T10:47:01.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:00 smithi175 ceph-mon[49304]: from='osd.4 [v2:172.21.15.175:6800/2327534492,v1:172.21.15.175:6801/2327534492]' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-21T10:47:01.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:00 smithi175 ceph-mon[49304]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-21T10:47:01.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:00 smithi175 ceph-mon[49304]: Health check update: Degraded data redundancy: 260/1689 objects degraded (15.394%), 28 pgs degraded (PG_DEGRADED) 2024-09-21T10:47:01.840 INFO:teuthology.orchestra.run.smithi123.stdout: 5 236406 9.83 MB/sec execute 482 sec latency 129.774 ms 2024-09-21T10:47:01.850 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:01 smithi123 ceph-mon[63856]: pgmap v220: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 352 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.4 KiB/s rd, 5.2 MiB/s wr, 543 op/s; 257/1641 objects degraded (15.661%) 2024-09-21T10:47:01.850 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:01 smithi123 ceph-mon[63856]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-21T10:47:01.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:01 smithi123 ceph-mon[63856]: from='osd.4 [v2:172.21.15.175:6800/2327534492,v1:172.21.15.175:6801/2327534492]' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi175", "root=default"]} : dispatch 2024-09-21T10:47:01.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:01 smithi123 ceph-mon[63856]: osdmap e135: 8 total, 7 up, 8 in 2024-09-21T10:47:01.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:01 smithi123 ceph-mon[63856]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi175", "root=default"]} : dispatch 2024-09-21T10:47:02.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:01 smithi175 ceph-mon[49304]: pgmap v220: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 352 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.4 KiB/s rd, 5.2 MiB/s wr, 543 op/s; 257/1641 objects degraded (15.661%) 2024-09-21T10:47:02.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:01 smithi175 ceph-mon[49304]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-21T10:47:02.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:01 smithi175 ceph-mon[49304]: from='osd.4 [v2:172.21.15.175:6800/2327534492,v1:172.21.15.175:6801/2327534492]' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi175", "root=default"]} : dispatch 2024-09-21T10:47:02.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:01 smithi175 ceph-mon[49304]: osdmap e135: 8 total, 7 up, 8 in 2024-09-21T10:47:02.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:01 smithi175 ceph-mon[49304]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi175", "root=default"]} : dispatch 2024-09-21T10:47:02.840 INFO:teuthology.orchestra.run.smithi123.stdout: 5 236444 9.82 MB/sec execute 483 sec latency 1105.918 ms 2024-09-21T10:47:03.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:02 smithi123 ceph-mon[63856]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T10:47:03.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:02 smithi123 ceph-mon[63856]: osd.4 [v2:172.21.15.175:6800/2327534492,v1:172.21.15.175:6801/2327534492] boot 2024-09-21T10:47:03.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:02 smithi123 ceph-mon[63856]: osdmap e136: 8 total, 8 up, 8 in 2024-09-21T10:47:03.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:02 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-21T10:47:03.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:02 smithi123 ceph-mon[63856]: osdmap e137: 8 total, 8 up, 8 in 2024-09-21T10:47:03.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:02 smithi175 ceph-mon[49304]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T10:47:03.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:02 smithi175 ceph-mon[49304]: osd.4 [v2:172.21.15.175:6800/2327534492,v1:172.21.15.175:6801/2327534492] boot 2024-09-21T10:47:03.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:02 smithi175 ceph-mon[49304]: osdmap e136: 8 total, 8 up, 8 in 2024-09-21T10:47:03.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:02 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-21T10:47:03.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:02 smithi175 ceph-mon[49304]: osdmap e137: 8 total, 8 up, 8 in 2024-09-21T10:47:03.840 INFO:teuthology.orchestra.run.smithi123.stdout: 5 236515 9.80 MB/sec execute 484 sec latency 929.768 ms 2024-09-21T10:47:03.994 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:03 smithi123 ceph-mon[63856]: pgmap v223: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 352 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.2 MiB/s wr, 402 op/s; 253/1587 objects degraded (15.942%) 2024-09-21T10:47:03.994 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:03 smithi123 ceph-mon[63856]: osdmap e138: 8 total, 8 up, 8 in 2024-09-21T10:47:04.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:03 smithi175 ceph-mon[49304]: pgmap v223: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 352 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.2 MiB/s wr, 402 op/s; 253/1587 objects degraded (15.942%) 2024-09-21T10:47:04.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:03 smithi175 ceph-mon[49304]: osdmap e138: 8 total, 8 up, 8 in 2024-09-21T10:47:04.840 INFO:teuthology.orchestra.run.smithi123.stdout: 5 236983 9.80 MB/sec execute 485 sec latency 64.436 ms 2024-09-21T10:47:05.823 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:05 smithi123 ceph-mon[63856]: pgmap v226: 97 pgs: 1 active+recovering+degraded, 8 remapped+peering, 3 active+recovery_wait+degraded, 11 active+undersized, 9 active+undersized+degraded, 65 active+clean; 348 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 2.8 MiB/s wr, 339 op/s; 148/1536 objects degraded (9.635%); 11 KiB/s, 54 keys/s, 5 objects/s recovering 2024-09-21T10:47:05.823 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:05 smithi123 ceph-mon[63856]: osdmap e139: 8 total, 8 up, 8 in 2024-09-21T10:47:05.823 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:05 smithi123 ceph-mon[63856]: Health check update: Degraded data redundancy: 148/1536 objects degraded (9.635%), 13 pgs degraded (PG_DEGRADED) 2024-09-21T10:47:05.824 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:05 smithi123 ceph-mon[63856]: osdmap e140: 8 total, 8 up, 8 in 2024-09-21T10:47:05.840 INFO:teuthology.orchestra.run.smithi123.stdout: 5 237296 9.81 MB/sec execute 486 sec latency 592.259 ms 2024-09-21T10:47:06.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:05 smithi175 ceph-mon[49304]: pgmap v226: 97 pgs: 1 active+recovering+degraded, 8 remapped+peering, 3 active+recovery_wait+degraded, 11 active+undersized, 9 active+undersized+degraded, 65 active+clean; 348 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 2.8 MiB/s wr, 339 op/s; 148/1536 objects degraded (9.635%); 11 KiB/s, 54 keys/s, 5 objects/s recovering 2024-09-21T10:47:06.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:05 smithi175 ceph-mon[49304]: osdmap e139: 8 total, 8 up, 8 in 2024-09-21T10:47:06.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:05 smithi175 ceph-mon[49304]: Health check update: Degraded data redundancy: 148/1536 objects degraded (9.635%), 13 pgs degraded (PG_DEGRADED) 2024-09-21T10:47:06.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:05 smithi175 ceph-mon[49304]: osdmap e140: 8 total, 8 up, 8 in 2024-09-21T10:47:06.840 INFO:teuthology.orchestra.run.smithi123.stdout: 5 237527 9.81 MB/sec execute 487 sec latency 799.137 ms 2024-09-21T10:47:07.097 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:07 smithi123 ceph-mon[63856]: osdmap e141: 8 total, 8 up, 8 in 2024-09-21T10:47:07.097 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:07 smithi123 ceph-mon[63856]: pgmap v230: 97 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded, 8 remapped+peering, 3 active+recovery_wait+degraded, 82 active+clean; 350 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 3.5 MiB/s wr, 295 op/s; 140/1530 objects degraded (9.150%); 90 KiB/s, 61 keys/s, 19 objects/s recovering 2024-09-21T10:47:07.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:07 smithi175 ceph-mon[49304]: osdmap e141: 8 total, 8 up, 8 in 2024-09-21T10:47:07.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:07 smithi175 ceph-mon[49304]: pgmap v230: 97 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded, 8 remapped+peering, 3 active+recovery_wait+degraded, 82 active+clean; 350 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 3.5 MiB/s wr, 295 op/s; 140/1530 objects degraded (9.150%); 90 KiB/s, 61 keys/s, 19 objects/s recovering 2024-09-21T10:47:07.841 INFO:teuthology.orchestra.run.smithi123.stdout: 5 237747 9.80 MB/sec execute 488 sec latency 811.626 ms 2024-09-21T10:47:08.125 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:08 smithi123 ceph-mon[63856]: osdmap e142: 8 total, 8 up, 8 in 2024-09-21T10:47:08.125 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:08 smithi123 ceph-mon[63856]: osdmap e143: 8 total, 8 up, 8 in 2024-09-21T10:47:08.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:08 smithi175 ceph-mon[49304]: osdmap e142: 8 total, 8 up, 8 in 2024-09-21T10:47:08.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:08 smithi175 ceph-mon[49304]: osdmap e143: 8 total, 8 up, 8 in 2024-09-21T10:47:08.840 INFO:teuthology.orchestra.run.smithi123.stdout: 5 237978 9.79 MB/sec execute 489 sec latency 1031.338 ms 2024-09-21T10:47:09.144 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:09 smithi123 ceph-mon[63856]: pgmap v233: 97 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded, 8 remapped+peering, 1 active+recovery_wait+degraded, 84 active+clean; 355 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 580 B/s rd, 3.7 MiB/s wr, 486 op/s; 102/1557 objects degraded (6.551%); 266 KiB/s, 24 objects/s recovering 2024-09-21T10:47:09.144 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:09 smithi123 ceph-mon[63856]: osdmap e144: 8 total, 8 up, 8 in 2024-09-21T10:47:09.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:09 smithi175 ceph-mon[49304]: pgmap v233: 97 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded, 8 remapped+peering, 1 active+recovery_wait+degraded, 84 active+clean; 355 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 580 B/s rd, 3.7 MiB/s wr, 486 op/s; 102/1557 objects degraded (6.551%); 266 KiB/s, 24 objects/s recovering 2024-09-21T10:47:09.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:09 smithi175 ceph-mon[49304]: osdmap e144: 8 total, 8 up, 8 in 2024-09-21T10:47:09.840 INFO:teuthology.orchestra.run.smithi123.stdout: 5 238160 9.77 MB/sec execute 490 sec latency 907.817 ms 2024-09-21T10:47:10.185 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:10 smithi123 ceph-mon[63856]: Health check update: Degraded data redundancy: 102/1557 objects degraded (6.551%), 5 pgs degraded (PG_DEGRADED) 2024-09-21T10:47:10.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:10 smithi175 ceph-mon[49304]: Health check update: Degraded data redundancy: 102/1557 objects degraded (6.551%), 5 pgs degraded (PG_DEGRADED) 2024-09-21T10:47:10.840 INFO:teuthology.orchestra.run.smithi123.stdout: 5 238361 9.77 MB/sec execute 491 sec latency 80.137 ms 2024-09-21T10:47:11.260 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:11 smithi123 ceph-mon[63856]: pgmap v235: 97 pgs: 2 peering, 1 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 92 active+clean; 357 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.2 KiB/s rd, 4.0 MiB/s wr, 391 op/s; 23/1635 objects degraded (1.407%); 1.3 MiB/s, 55 objects/s recovering 2024-09-21T10:47:11.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:11 smithi175 ceph-mon[49304]: pgmap v235: 97 pgs: 2 peering, 1 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 92 active+clean; 357 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.2 KiB/s rd, 4.0 MiB/s wr, 391 op/s; 23/1635 objects degraded (1.407%); 1.3 MiB/s, 55 objects/s recovering 2024-09-21T10:47:11.840 INFO:teuthology.orchestra.run.smithi123.stdout: 5 238532 9.76 MB/sec execute 492 sec latency 62.821 ms 2024-09-21T10:47:12.534 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:12 smithi123 ceph-mon[63856]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 23/1635 objects degraded (1.407%), 1 pg degraded) 2024-09-21T10:47:12.534 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:12 smithi123 ceph-mon[63856]: Cluster is now healthy 2024-09-21T10:47:12.841 INFO:teuthology.orchestra.run.smithi123.stdout: 5 238823 9.75 MB/sec execute 493 sec latency 68.905 ms 2024-09-21T10:47:12.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:12 smithi175 ceph-mon[49304]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 23/1635 objects degraded (1.407%), 1 pg degraded) 2024-09-21T10:47:12.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:12 smithi175 ceph-mon[49304]: Cluster is now healthy 2024-09-21T10:47:13.841 INFO:teuthology.orchestra.run.smithi123.stdout: 5 239250 9.74 MB/sec execute 494 sec latency 52.717 ms 2024-09-21T10:47:13.849 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:13 smithi123 ceph-mon[63856]: pgmap v236: 97 pgs: 2 peering, 95 active+clean; 357 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 3.0 KiB/s rd, 3.8 MiB/s wr, 313 op/s; 1.1 MiB/s, 49 objects/s recovering 2024-09-21T10:47:13.849 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:13 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:47:13.849 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:13 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:47:13.849 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:13 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-21T10:47:13.849 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:13 smithi123 ceph-mon[63856]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-21T10:47:13.849 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:13 smithi123 ceph-mon[63856]: Upgrade: osd.5 is safe to restart 2024-09-21T10:47:13.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:13 smithi175 ceph-mon[49304]: pgmap v236: 97 pgs: 2 peering, 95 active+clean; 357 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 3.0 KiB/s rd, 3.8 MiB/s wr, 313 op/s; 1.1 MiB/s, 49 objects/s recovering 2024-09-21T10:47:13.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:13 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:47:13.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:13 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:47:13.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:13 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-21T10:47:13.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:13 smithi175 ceph-mon[49304]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-21T10:47:13.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:13 smithi175 ceph-mon[49304]: Upgrade: osd.5 is safe to restart 2024-09-21T10:47:14.840 INFO:teuthology.orchestra.run.smithi123.stdout: 5 239504 9.73 MB/sec execute 495 sec latency 60.460 ms 2024-09-21T10:47:15.395 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:15 smithi123 ceph-mon[63856]: Upgrade: Updating osd.5 2024-09-21T10:47:15.395 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:15 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:47:15.395 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:15 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-21T10:47:15.395 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:15 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:47:15.395 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:15 smithi123 ceph-mon[63856]: Deploying daemon osd.5 on smithi175 2024-09-21T10:47:15.395 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:15 smithi123 ceph-mon[63856]: pgmap v237: 97 pgs: 97 active+clean; 354 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.5 MiB/s wr, 461 op/s; 893 KiB/s, 41 objects/s recovering 2024-09-21T10:47:15.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:15 smithi175 ceph-mon[49304]: Upgrade: Updating osd.5 2024-09-21T10:47:15.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:15 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:47:15.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:15 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-21T10:47:15.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:15 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:47:15.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:15 smithi175 ceph-mon[49304]: Deploying daemon osd.5 on smithi175 2024-09-21T10:47:15.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:15 smithi175 ceph-mon[49304]: pgmap v237: 97 pgs: 97 active+clean; 354 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.5 MiB/s wr, 461 op/s; 893 KiB/s, 41 objects/s recovering 2024-09-21T10:47:15.841 INFO:teuthology.orchestra.run.smithi123.stdout: 5 239774 9.74 MB/sec execute 496 sec latency 55.853 ms 2024-09-21T10:47:16.841 INFO:teuthology.orchestra.run.smithi123.stdout: 5 240153 9.74 MB/sec execute 497 sec latency 54.376 ms 2024-09-21T10:47:17.495 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:17 smithi123 ceph-mon[63856]: pgmap v238: 97 pgs: 97 active+clean; 351 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.8 MiB/s wr, 452 op/s; 785 KiB/s, 36 objects/s recovering 2024-09-21T10:47:17.572 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:17 smithi175 ceph-mon[49304]: pgmap v238: 97 pgs: 97 active+clean; 351 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.8 MiB/s wr, 452 op/s; 785 KiB/s, 36 objects/s recovering 2024-09-21T10:47:17.841 INFO:teuthology.orchestra.run.smithi123.stdout: 5 240562 9.75 MB/sec execute 498 sec latency 59.611 ms 2024-09-21T10:47:17.862 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:47:17 smithi175 systemd[1]: Stopping Ceph osd.5 for 3ccd15e6-7804-11ef-baf6-efdc52797490... 2024-09-21T10:47:18.123 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:47:17 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-5[32878]: 2024-09-21T10:47:17.859+0000 7f9188e7d700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:47:18.123 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:47:17 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-5[32878]: 2024-09-21T10:47:17.859+0000 7f9188e7d700 -1 osd.5 144 *** Got signal Terminated *** 2024-09-21T10:47:18.123 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:47:17 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-5[32878]: 2024-09-21T10:47:17.859+0000 7f9188e7d700 -1 osd.5 144 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T10:47:18.353 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:18 smithi123 ceph-mon[63856]: osd.5 marked itself down and dead 2024-09-21T10:47:18.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:18 smithi175 ceph-mon[49304]: osd.5 marked itself down and dead 2024-09-21T10:47:18.841 INFO:teuthology.orchestra.run.smithi123.stdout: 5 240773 9.74 MB/sec execute 499 sec latency 497.580 ms 2024-09-21T10:47:19.262 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:47:18 smithi175 podman[54834]: 2024-09-21 10:47:18.980398488 +0000 UTC m=+1.230835638 container died e270d5a21b1443aee73d3a15a99a4ba58ab98d1d9837945fc86a7f03650a1bee (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-5, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, RELEASE=HEAD, ceph=True, com.redhat.component=centos-stream-container, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , io.openshift.expose-services=) 2024-09-21T10:47:19.589 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:19 smithi123 ceph-mon[63856]: pgmap v239: 97 pgs: 97 active+clean; 354 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.7 MiB/s wr, 344 op/s; 581 KiB/s, 26 objects/s recovering 2024-09-21T10:47:19.589 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:19 smithi123 ceph-mon[63856]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T10:47:19.589 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:19 smithi123 ceph-mon[63856]: osdmap e145: 8 total, 7 up, 8 in 2024-09-21T10:47:19.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:19 smithi175 ceph-mon[49304]: pgmap v239: 97 pgs: 97 active+clean; 354 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.7 MiB/s wr, 344 op/s; 581 KiB/s, 26 objects/s recovering 2024-09-21T10:47:19.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:19 smithi175 ceph-mon[49304]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T10:47:19.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:19 smithi175 ceph-mon[49304]: osdmap e145: 8 total, 7 up, 8 in 2024-09-21T10:47:19.623 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:47:19 smithi175 podman[54834]: 2024-09-21 10:47:19.343182915 +0000 UTC m=+1.593620074 container cleanup e270d5a21b1443aee73d3a15a99a4ba58ab98d1d9837945fc86a7f03650a1bee (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-5, distribution-scope=public, io.openshift.tags=base centos centos-stream, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, version=8, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, ceph=True, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , release=754, GIT_BRANCH=HEAD, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0) 2024-09-21T10:47:19.624 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:47:19 smithi175 bash[54834]: ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-5 2024-09-21T10:47:19.841 INFO:teuthology.orchestra.run.smithi123.stdout: 5 240856 9.73 MB/sec execute 500 sec latency 1073.018 ms 2024-09-21T10:47:20.024 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:47:19 smithi175 podman[54852]: 2024-09-21 10:47:19.686458226 +0000 UTC m=+0.697074023 container remove e270d5a21b1443aee73d3a15a99a4ba58ab98d1d9837945fc86a7f03650a1bee (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-5, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, release=754, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, com.redhat.component=centos-stream-container, distribution-scope=public, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-09-21T10:47:20.317 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:47:20 smithi175 podman[54938]: 2024-09-21 10:47:19.9255079 +0000 UTC m=+0.017807365 image pull 8f4d013db793795fef09ff5edf96e6b42ba8cd2c9158b2f0b7a5d7c1e657c063 quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472 2024-09-21T10:47:20.317 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:47:20 smithi175 podman[54938]: 2024-09-21 10:47:20.071004719 +0000 UTC m=+0.163304184 container create 4dfbd1c5fd929e42f95dccf4d86ab7d339aadce125176d5c3169ba975c569261 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-5-deactivate, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, org.label-schema.vendor=CentOS) 2024-09-21T10:47:20.317 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:47:20 smithi175 podman[54938]: 2024-09-21 10:47:20.315573432 +0000 UTC m=+0.407872906 container init 4dfbd1c5fd929e42f95dccf4d86ab7d339aadce125176d5c3169ba975c569261 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-5-deactivate, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS) 2024-09-21T10:47:20.358 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:20 smithi123 ceph-mon[63856]: osdmap e146: 8 total, 7 up, 8 in 2024-09-21T10:47:20.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:20 smithi175 ceph-mon[49304]: osdmap e146: 8 total, 7 up, 8 in 2024-09-21T10:47:20.624 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:47:20 smithi175 podman[54938]: 2024-09-21 10:47:20.320219308 +0000 UTC m=+0.412518773 container start 4dfbd1c5fd929e42f95dccf4d86ab7d339aadce125176d5c3169ba975c569261 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-5-deactivate, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-09-21T10:47:20.624 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:47:20 smithi175 podman[54938]: 2024-09-21 10:47:20.407254431 +0000 UTC m=+0.499553911 container attach 4dfbd1c5fd929e42f95dccf4d86ab7d339aadce125176d5c3169ba975c569261 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-5-deactivate, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2) 2024-09-21T10:47:20.624 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:47:20 smithi175 podman[54938]: 2024-09-21 10:47:20.53424415 +0000 UTC m=+0.626543617 container died 4dfbd1c5fd929e42f95dccf4d86ab7d339aadce125176d5c3169ba975c569261 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-5-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, maintainer=Guillaume Abrioux ) 2024-09-21T10:47:20.841 INFO:teuthology.orchestra.run.smithi123.stdout: 5 241019 9.72 MB/sec execute 501 sec latency 98.405 ms 2024-09-21T10:47:21.401 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:21 smithi123 ceph-mon[63856]: pgmap v242: 97 pgs: 1 active+undersized, 1 peering, 2 active+undersized+degraded, 93 active+clean; 349 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 4.3 MiB/s wr, 493 op/s; 21/1590 objects degraded (1.321%) 2024-09-21T10:47:21.401 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:21 smithi123 ceph-mon[63856]: Health check failed: Degraded data redundancy: 21/1590 objects degraded (1.321%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T10:47:21.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:21 smithi175 ceph-mon[49304]: pgmap v242: 97 pgs: 1 active+undersized, 1 peering, 2 active+undersized+degraded, 93 active+clean; 349 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 4.3 MiB/s wr, 493 op/s; 21/1590 objects degraded (1.321%) 2024-09-21T10:47:21.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:21 smithi175 ceph-mon[49304]: Health check failed: Degraded data redundancy: 21/1590 objects degraded (1.321%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T10:47:21.623 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:47:21 smithi175 podman[54961]: 2024-09-21 10:47:21.161623407 +0000 UTC m=+0.616476403 container remove 4dfbd1c5fd929e42f95dccf4d86ab7d339aadce125176d5c3169ba975c569261 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-5-deactivate, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T10:47:21.624 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:47:21 smithi175 systemd[1]: ceph-3ccd15e6-7804-11ef-baf6-efdc52797490@osd.5.service: Deactivated successfully. 2024-09-21T10:47:21.624 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:47:21 smithi175 systemd[1]: Stopped Ceph osd.5 for 3ccd15e6-7804-11ef-baf6-efdc52797490. 2024-09-21T10:47:21.624 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:47:21 smithi175 systemd[1]: ceph-3ccd15e6-7804-11ef-baf6-efdc52797490@osd.5.service: Consumed 5min 1.862s CPU time. 2024-09-21T10:47:21.624 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:47:21 smithi175 systemd[1]: Starting Ceph osd.5 for 3ccd15e6-7804-11ef-baf6-efdc52797490... 2024-09-21T10:47:21.841 INFO:teuthology.orchestra.run.smithi123.stdout: 5 241374 9.72 MB/sec execute 502 sec latency 99.462 ms 2024-09-21T10:47:21.873 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:47:21 smithi175 podman[55058]: 2024-09-21 10:47:21.524397493 +0000 UTC m=+0.017152806 image pull 8f4d013db793795fef09ff5edf96e6b42ba8cd2c9158b2f0b7a5d7c1e657c063 quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472 2024-09-21T10:47:21.873 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:47:21 smithi175 podman[55058]: 2024-09-21 10:47:21.64796795 +0000 UTC m=+0.140723254 container create 470c596d4fed580b30fc28d47bc3f6df9a816d2b1c971584825e9463c2fda757 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-5-activate, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-09-21T10:47:22.374 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:47:21 smithi175 podman[55058]: 2024-09-21 10:47:21.907579374 +0000 UTC m=+0.400334686 container init 470c596d4fed580b30fc28d47bc3f6df9a816d2b1c971584825e9463c2fda757 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-5-activate, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True) 2024-09-21T10:47:22.374 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:47:21 smithi175 podman[55058]: 2024-09-21 10:47:21.911316598 +0000 UTC m=+0.404071909 container start 470c596d4fed580b30fc28d47bc3f6df9a816d2b1c971584825e9463c2fda757 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-5-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3) 2024-09-21T10:47:22.374 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:47:21 smithi175 podman[55058]: 2024-09-21 10:47:21.999267141 +0000 UTC m=+0.492022449 container attach 470c596d4fed580b30fc28d47bc3f6df9a816d2b1c971584825e9463c2fda757 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-5-activate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3) 2024-09-21T10:47:22.374 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:47:22 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-5-activate[55073]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:47:22.374 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:47:22 smithi175 bash[55058]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:47:22.374 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:47:22 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-5-activate[55073]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:47:22.374 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:47:22 smithi175 bash[55058]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:47:22.841 INFO:teuthology.orchestra.run.smithi123.stdout: 5 241739 9.72 MB/sec execute 503 sec latency 42.121 ms 2024-09-21T10:47:23.703 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:23 smithi123 ceph-mon[63856]: pgmap v243: 97 pgs: 2 active+undersized, 1 peering, 10 active+undersized+degraded, 84 active+clean; 352 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 3.7 MiB/s wr, 338 op/s; 86/1626 objects degraded (5.289%); 8.0 KiB/s, 0 objects/s recovering 2024-09-21T10:47:23.703 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:23 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:47:23.841 INFO:teuthology.orchestra.run.smithi123.stdout: 5 242012 9.71 MB/sec execute 504 sec latency 45.040 ms 2024-09-21T10:47:24.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:23 smithi175 ceph-mon[49304]: pgmap v243: 97 pgs: 2 active+undersized, 1 peering, 10 active+undersized+degraded, 84 active+clean; 352 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 3.7 MiB/s wr, 338 op/s; 86/1626 objects degraded (5.289%); 8.0 KiB/s, 0 objects/s recovering 2024-09-21T10:47:24.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:23 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:47:24.841 INFO:teuthology.orchestra.run.smithi123.stdout: 5 242335 9.71 MB/sec execute 505 sec latency 43.977 ms 2024-09-21T10:47:25.373 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:47:25 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-5-activate[55073]: --> Failed to activate via raw: did not find any matching OSD to activate 2024-09-21T10:47:25.373 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:47:25 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-5-activate[55073]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:47:25.374 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:47:25 smithi175 bash[55058]: --> Failed to activate via raw: did not find any matching OSD to activate 2024-09-21T10:47:25.374 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:47:25 smithi175 bash[55058]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:47:25.374 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:47:25 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-5-activate[55073]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:47:25.374 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:47:25 smithi175 bash[55058]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:47:25.374 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:47:25 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-5-activate[55073]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-21T10:47:25.374 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:47:25 smithi175 bash[55058]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-21T10:47:25.374 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:47:25 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-5-activate[55073]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-5 --no-mon-config 2024-09-21T10:47:25.374 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:47:25 smithi175 bash[55058]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-5 --no-mon-config 2024-09-21T10:47:25.699 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:47:25 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-5-activate[55073]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-5/block 2024-09-21T10:47:25.699 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:47:25 smithi175 bash[55058]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-5/block 2024-09-21T10:47:25.699 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:47:25 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-5-activate[55073]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-5/block 2024-09-21T10:47:25.700 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:47:25 smithi175 bash[55058]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-5/block 2024-09-21T10:47:25.700 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:47:25 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-5-activate[55073]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-21T10:47:25.700 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:47:25 smithi175 bash[55058]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-21T10:47:25.700 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:47:25 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-5-activate[55073]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-21T10:47:25.700 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:47:25 smithi175 bash[55058]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-21T10:47:25.700 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:47:25 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-5-activate[55073]: --> ceph-volume lvm activate successful for osd ID: 5 2024-09-21T10:47:25.700 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:47:25 smithi175 bash[55058]: --> ceph-volume lvm activate successful for osd ID: 5 2024-09-21T10:47:25.700 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:47:25 smithi175 podman[55058]: 2024-09-21 10:47:25.415916338 +0000 UTC m=+3.908671647 container died 470c596d4fed580b30fc28d47bc3f6df9a816d2b1c971584825e9463c2fda757 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-5-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True) 2024-09-21T10:47:25.733 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:25 smithi123 ceph-mon[63856]: pgmap v244: 97 pgs: 7 active+undersized, 23 active+undersized+degraded, 67 active+clean; 355 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.4 MiB/s wr, 476 op/s; 181/1665 objects degraded (10.871%); 8.0 KiB/s, 0 objects/s recovering 2024-09-21T10:47:25.841 INFO:teuthology.orchestra.run.smithi123.stdout: 5 242694 9.70 MB/sec execute 506 sec latency 47.285 ms 2024-09-21T10:47:26.115 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:25 smithi175 ceph-mon[49304]: pgmap v244: 97 pgs: 7 active+undersized, 23 active+undersized+degraded, 67 active+clean; 355 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.4 MiB/s wr, 476 op/s; 181/1665 objects degraded (10.871%); 8.0 KiB/s, 0 objects/s recovering 2024-09-21T10:47:26.373 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:47:26 smithi175 podman[55281]: 2024-09-21 10:47:26.11403165 +0000 UTC m=+0.691786857 container remove 470c596d4fed580b30fc28d47bc3f6df9a816d2b1c971584825e9463c2fda757 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-5-activate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, ceph=True, org.label-schema.build-date=20240731, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T10:47:26.373 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:47:26 smithi175 podman[55335]: 2024-09-21 10:47:26.239818721 +0000 UTC m=+0.022263443 image pull 8f4d013db793795fef09ff5edf96e6b42ba8cd2c9158b2f0b7a5d7c1e657c063 quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472 2024-09-21T10:47:26.628 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:47:26 smithi175 podman[55335]: 2024-09-21 10:47:26.373423234 +0000 UTC m=+0.155867935 container create 8199a0b362df6545a0b031557ce2948aa33d9cb647d1549c9861e25558caf68f (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-5, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2) 2024-09-21T10:47:26.628 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:47:26 smithi175 podman[55335]: 2024-09-21 10:47:26.626494106 +0000 UTC m=+0.408938822 container init 8199a0b362df6545a0b031557ce2948aa33d9cb647d1549c9861e25558caf68f (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-5, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-09-21T10:47:26.841 INFO:teuthology.orchestra.run.smithi123.stdout: 5 243030 9.69 MB/sec execute 507 sec latency 52.859 ms 2024-09-21T10:47:27.000 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:26 smithi123 ceph-mon[63856]: Health check update: Degraded data redundancy: 183/1629 objects degraded (11.234%), 23 pgs degraded (PG_DEGRADED) 2024-09-21T10:47:27.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:26 smithi175 ceph-mon[49304]: Health check update: Degraded data redundancy: 183/1629 objects degraded (11.234%), 23 pgs degraded (PG_DEGRADED) 2024-09-21T10:47:27.124 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:47:26 smithi175 podman[55335]: 2024-09-21 10:47:26.629805881 +0000 UTC m=+0.412250585 container start 8199a0b362df6545a0b031557ce2948aa33d9cb647d1549c9861e25558caf68f (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-5, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T10:47:27.124 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:47:26 smithi175 ceph-osd[55353]: -- 172.21.15.175:0/1237427010 <== mon.1 v2:172.21.15.175:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55f3ca61a1e0 con 0x55f3c97bdc00 2024-09-21T10:47:27.124 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:47:26 smithi175 bash[55335]: 8199a0b362df6545a0b031557ce2948aa33d9cb647d1549c9861e25558caf68f 2024-09-21T10:47:27.124 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:47:26 smithi175 systemd[1]: Started Ceph osd.5 for 3ccd15e6-7804-11ef-baf6-efdc52797490. 2024-09-21T10:47:27.545 INFO:teuthology.orchestra.run.smithi123.stdout:true 2024-09-21T10:47:27.841 INFO:teuthology.orchestra.run.smithi123.stdout: 5 243248 9.69 MB/sec execute 508 sec latency 48.935 ms 2024-09-21T10:47:28.014 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:27 smithi123 ceph-mon[63856]: pgmap v245: 97 pgs: 7 active+undersized, 23 active+undersized+degraded, 67 active+clean; 353 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.0 MiB/s wr, 426 op/s; 183/1629 objects degraded (11.234%); 8.0 KiB/s, 0 objects/s recovering 2024-09-21T10:47:28.014 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:27 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:47:28.014 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:27 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:47:28.015 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:27 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:47:28.015 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:27 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:47:28.015 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:27 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:47:28.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:27 smithi175 ceph-mon[49304]: pgmap v245: 97 pgs: 7 active+undersized, 23 active+undersized+degraded, 67 active+clean; 353 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.0 MiB/s wr, 426 op/s; 183/1629 objects degraded (11.234%); 8.0 KiB/s, 0 objects/s recovering 2024-09-21T10:47:28.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:27 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:47:28.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:27 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:47:28.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:27 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:47:28.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:27 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:47:28.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:27 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:47:28.372 INFO:teuthology.orchestra.run.smithi123.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T10:47:28.373 INFO:teuthology.orchestra.run.smithi123.stdout:alertmanager.smithi123 smithi123 *:9093,9094 running (7m) 76s ago 17m 22.0M - 0.27.0 11f11916f8cd 5d550bd97de5 2024-09-21T10:47:28.373 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi123 smithi123 running (4m) 76s ago 17m 6899k - 19.3.0-5103-g5c1cdc3c 8f4d013db793 c5e1ef39a0a8 2024-09-21T10:47:28.373 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi175 smithi175 running (4m) 35s ago 16m 6895k - 19.3.0-5103-g5c1cdc3c 8f4d013db793 fb1ca999ccfd 2024-09-21T10:47:28.373 INFO:teuthology.orchestra.run.smithi123.stdout:grafana.smithi123 smithi123 *:3000 running (7m) 76s ago 16m 52.6M - dad864ee21e9 771df81763ba 2024-09-21T10:47:28.373 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi123.uqoyow smithi123 running (10m) 76s ago 10m 41.2M - 17.2.0 e1d6a67b021e f0a21eaac266 2024-09-21T10:47:28.373 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi175.jrlevw smithi175 running (10m) 35s ago 10m 14.3M - 17.2.0 e1d6a67b021e b21b16f4bc50 2024-09-21T10:47:28.373 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi123.stnuit smithi123 *:8443,9283,8765 running (7m) 76s ago 18m 519M - 19.3.0-5103-g5c1cdc3c 8f4d013db793 6812cc5cb56b 2024-09-21T10:47:28.373 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi175.txvwrq smithi175 *:8443,9283,8765 running (6m) 35s ago 16m 456M - 19.3.0-5103-g5c1cdc3c 8f4d013db793 304f0408075e 2024-09-21T10:47:28.373 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi123 smithi123 running (5m) 76s ago 18m 52.0M 2048M 19.3.0-5103-g5c1cdc3c 8f4d013db793 6b95207a2d80 2024-09-21T10:47:28.373 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi175 smithi175 running (5m) 35s ago 16m 42.0M 2048M 19.3.0-5103-g5c1cdc3c 8f4d013db793 3abee2d37f0f 2024-09-21T10:47:28.373 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.0.0.smithi123.ddatpr smithi123 *:2049 running (10m) 76s ago 10m 281M - 4.0 e1d6a67b021e 36f8cb2281eb 2024-09-21T10:47:28.373 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.1.0.smithi175.wpawer smithi175 *:2049 running (10m) 35s ago 10m 90.7M - 4.0 e1d6a67b021e 2200aeed9200 2024-09-21T10:47:28.374 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi123 smithi123 *:9100 running (7m) 76s ago 16m 12.3M - 1.7.0 72c9c2088986 ca7d0b29af81 2024-09-21T10:47:28.374 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi175 smithi175 *:9100 running (7m) 35s ago 16m 10.6M - 1.7.0 72c9c2088986 46c4f6b7c149 2024-09-21T10:47:28.374 INFO:teuthology.orchestra.run.smithi123.stdout:osd.0 smithi123 running (3m) 76s ago 15m 362M 1936M 19.3.0-5103-g5c1cdc3c 8f4d013db793 cc40f695c3a8 2024-09-21T10:47:28.374 INFO:teuthology.orchestra.run.smithi123.stdout:osd.1 smithi123 running (3m) 76s ago 14m 279M 1936M 19.3.0-5103-g5c1cdc3c 8f4d013db793 1be249025c7f 2024-09-21T10:47:28.374 INFO:teuthology.orchestra.run.smithi123.stdout:osd.2 smithi123 running (2m) 76s ago 14m 232M 1936M 19.3.0-5103-g5c1cdc3c 8f4d013db793 647aec759e3c 2024-09-21T10:47:28.374 INFO:teuthology.orchestra.run.smithi123.stdout:osd.3 smithi123 running (85s) 76s ago 14m 11.6M 1936M 19.3.0-5103-g5c1cdc3c 8f4d013db793 7a082fcad8e8 2024-09-21T10:47:28.374 INFO:teuthology.orchestra.run.smithi123.stdout:osd.4 smithi175 running (39s) 35s ago 13m 11.5M 2704M 19.3.0-5103-g5c1cdc3c 8f4d013db793 b650f72d10f0 2024-09-21T10:47:28.374 INFO:teuthology.orchestra.run.smithi123.stdout:osd.5 smithi175 starting - - - 2704M 2024-09-21T10:47:28.374 INFO:teuthology.orchestra.run.smithi123.stdout:osd.6 smithi175 running (12m) 35s ago 12m 488M 2704M 17.2.0 e1d6a67b021e 9bcbf018eb20 2024-09-21T10:47:28.374 INFO:teuthology.orchestra.run.smithi123.stdout:osd.7 smithi175 running (12m) 35s ago 12m 707M 2704M 17.2.0 e1d6a67b021e e9b8f1f90994 2024-09-21T10:47:28.374 INFO:teuthology.orchestra.run.smithi123.stdout:prometheus.smithi123 smithi123 *:9095 running (6m) 76s ago 15m 55.4M - 2.51.0 1d3b7f56885b 37f944c9de88 2024-09-21T10:47:28.596 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:47:28 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-5[55349]: 2024-09-21T10:47:28.254+0000 7f2561123640 -1 Falling back to public interface 2024-09-21T10:47:28.841 INFO:teuthology.orchestra.run.smithi123.stdout: 5 243487 9.69 MB/sec execute 509 sec latency 44.021 ms 2024-09-21T10:47:28.868 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-09-21T10:47:28.868 INFO:teuthology.orchestra.run.smithi123.stdout: "mon": { 2024-09-21T10:47:28.868 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev)": 2 2024-09-21T10:47:28.868 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:47:28.868 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr": { 2024-09-21T10:47:28.868 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev)": 2 2024-09-21T10:47:28.868 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:47:28.869 INFO:teuthology.orchestra.run.smithi123.stdout: "osd": { 2024-09-21T10:47:28.869 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-09-21T10:47:28.869 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev)": 5 2024-09-21T10:47:28.869 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:47:28.869 INFO:teuthology.orchestra.run.smithi123.stdout: "mds": { 2024-09-21T10:47:28.869 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T10:47:28.869 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:47:28.869 INFO:teuthology.orchestra.run.smithi123.stdout: "overall": { 2024-09-21T10:47:28.869 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-09-21T10:47:28.869 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev)": 9 2024-09-21T10:47:28.869 INFO:teuthology.orchestra.run.smithi123.stdout: } 2024-09-21T10:47:28.869 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-09-21T10:47:28.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:28 smithi175 ceph-mon[49304]: from='client.34384 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:47:29.033 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:28 smithi123 ceph-mon[63856]: from='client.34384 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:47:29.288 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-09-21T10:47:29.288 INFO:teuthology.orchestra.run.smithi123.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472", 2024-09-21T10:47:29.288 INFO:teuthology.orchestra.run.smithi123.stdout: "in_progress": true, 2024-09-21T10:47:29.288 INFO:teuthology.orchestra.run.smithi123.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T10:47:29.288 INFO:teuthology.orchestra.run.smithi123.stdout: "services_complete": [ 2024-09-21T10:47:29.288 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr", 2024-09-21T10:47:29.288 INFO:teuthology.orchestra.run.smithi123.stdout: "crash", 2024-09-21T10:47:29.289 INFO:teuthology.orchestra.run.smithi123.stdout: "mon" 2024-09-21T10:47:29.289 INFO:teuthology.orchestra.run.smithi123.stdout: ], 2024-09-21T10:47:29.289 INFO:teuthology.orchestra.run.smithi123.stdout: "progress": "11/23 daemons upgraded", 2024-09-21T10:47:29.289 INFO:teuthology.orchestra.run.smithi123.stdout: "message": "Currently upgrading osd daemons", 2024-09-21T10:47:29.289 INFO:teuthology.orchestra.run.smithi123.stdout: "is_paused": false 2024-09-21T10:47:29.289 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-09-21T10:47:29.800 INFO:teuthology.orchestra.run.smithi123.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 181/1614 objects degraded (11.214%), 23 pgs degraded 2024-09-21T10:47:29.800 INFO:teuthology.orchestra.run.smithi123.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-21T10:47:29.801 INFO:teuthology.orchestra.run.smithi123.stdout: osd.5 (root=default,host=smithi175) is down 2024-09-21T10:47:29.801 INFO:teuthology.orchestra.run.smithi123.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 181/1614 objects degraded (11.214%), 23 pgs degraded 2024-09-21T10:47:29.801 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.2 is active+undersized+degraded, acting [1,6] 2024-09-21T10:47:29.801 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.3 is active+undersized+degraded, acting [2,7] 2024-09-21T10:47:29.801 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.b is active+undersized+degraded, acting [7,4] 2024-09-21T10:47:29.801 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.c is active+undersized+degraded, acting [2,0] 2024-09-21T10:47:29.801 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.12 is active+undersized+degraded, acting [3,7] 2024-09-21T10:47:29.801 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.14 is active+undersized+degraded, acting [6,3] 2024-09-21T10:47:29.801 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.15 is active+undersized+degraded, acting [1,0] 2024-09-21T10:47:29.801 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.16 is active+undersized+degraded, acting [6,2] 2024-09-21T10:47:29.801 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.17 is active+undersized+degraded, acting [6,2] 2024-09-21T10:47:29.802 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.18 is active+undersized+degraded, acting [4,7] 2024-09-21T10:47:29.802 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.1c is active+undersized+degraded, acting [7,2] 2024-09-21T10:47:29.802 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.1e is active+undersized+degraded, acting [3,0] 2024-09-21T10:47:29.802 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.2 is active+undersized+degraded, acting [3,6] 2024-09-21T10:47:29.802 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.4 is active+undersized+degraded, acting [1,2] 2024-09-21T10:47:29.802 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.5 is active+undersized+degraded, acting [3,2] 2024-09-21T10:47:29.802 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.c is active+undersized+degraded, acting [3,6] 2024-09-21T10:47:29.802 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.d is active+undersized+degraded, acting [7,6] 2024-09-21T10:47:29.802 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.10 is active+undersized+degraded, acting [6,0] 2024-09-21T10:47:29.802 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.16 is active+undersized+degraded, acting [7,1] 2024-09-21T10:47:29.802 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.17 is active+undersized+degraded, acting [0,3] 2024-09-21T10:47:29.803 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.1c is active+undersized+degraded, acting [4,1] 2024-09-21T10:47:29.803 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.1d is active+undersized+degraded, acting [4,6] 2024-09-21T10:47:29.803 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.1f is active+undersized+degraded, acting [0,2] 2024-09-21T10:47:29.841 INFO:teuthology.orchestra.run.smithi123.stdout: 5 243842 9.69 MB/sec execute 510 sec latency 72.913 ms 2024-09-21T10:47:30.043 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:29 smithi123 ceph-mon[63856]: from='client.34388 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:47:30.043 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:29 smithi123 ceph-mon[63856]: pgmap v246: 97 pgs: 7 active+undersized, 23 active+undersized+degraded, 67 active+clean; 347 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.7 KiB/s rd, 3.4 MiB/s wr, 254 op/s; 181/1614 objects degraded (11.214%); 6.4 KiB/s, 0 objects/s recovering 2024-09-21T10:47:30.043 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:29 smithi123 ceph-mon[63856]: from='client.34392 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:47:30.043 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:29 smithi123 ceph-mon[63856]: from='client.? 172.21.15.123:0/2730437259' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:47:30.043 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:29 smithi123 ceph-mon[63856]: from='client.? 172.21.15.123:0/2134145500' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T10:47:30.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:29 smithi175 ceph-mon[49304]: from='client.34388 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:47:30.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:29 smithi175 ceph-mon[49304]: pgmap v246: 97 pgs: 7 active+undersized, 23 active+undersized+degraded, 67 active+clean; 347 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.7 KiB/s rd, 3.4 MiB/s wr, 254 op/s; 181/1614 objects degraded (11.214%); 6.4 KiB/s, 0 objects/s recovering 2024-09-21T10:47:30.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:29 smithi175 ceph-mon[49304]: from='client.34392 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:47:30.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:29 smithi175 ceph-mon[49304]: from='client.? 172.21.15.123:0/2730437259' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:47:30.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:29 smithi175 ceph-mon[49304]: from='client.? 172.21.15.123:0/2134145500' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T10:47:30.830 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:30 smithi123 ceph-mon[63856]: from='client.34400 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:47:30.841 INFO:teuthology.orchestra.run.smithi123.stdout: 5 244140 9.68 MB/sec execute 511 sec latency 93.497 ms 2024-09-21T10:47:31.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:30 smithi175 ceph-mon[49304]: from='client.34400 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:47:31.841 INFO:teuthology.orchestra.run.smithi123.stdout: 5 244400 9.69 MB/sec execute 512 sec latency 67.961 ms 2024-09-21T10:47:31.871 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:31 smithi123 ceph-mon[63856]: pgmap v247: 97 pgs: 7 active+undersized, 23 active+undersized+degraded, 67 active+clean; 348 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.3 MiB/s wr, 390 op/s; 171/1563 objects degraded (10.940%); 5.8 KiB/s, 0 objects/s recovering 2024-09-21T10:47:31.871 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:31 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:47:31.871 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:31 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:47:32.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:31 smithi175 ceph-mon[49304]: pgmap v247: 97 pgs: 7 active+undersized, 23 active+undersized+degraded, 67 active+clean; 348 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.3 MiB/s wr, 390 op/s; 171/1563 objects degraded (10.940%); 5.8 KiB/s, 0 objects/s recovering 2024-09-21T10:47:32.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:31 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:47:32.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:31 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:47:32.841 INFO:teuthology.orchestra.run.smithi123.stdout: 5 244593 9.68 MB/sec execute 513 sec latency 56.247 ms 2024-09-21T10:47:33.179 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:33 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:47:33.179 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:33 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:47:33.179 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:33 smithi123 ceph-mon[63856]: pgmap v248: 97 pgs: 7 active+undersized, 23 active+undersized+degraded, 67 active+clean; 349 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.0 MiB/s wr, 368 op/s; 175/1569 objects degraded (11.154%); 5.3 KiB/s, 0 objects/s recovering 2024-09-21T10:47:33.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:33 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:47:33.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:33 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:47:33.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:33 smithi175 ceph-mon[49304]: pgmap v248: 97 pgs: 7 active+undersized, 23 active+undersized+degraded, 67 active+clean; 349 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.0 MiB/s wr, 368 op/s; 175/1569 objects degraded (11.154%); 5.3 KiB/s, 0 objects/s recovering 2024-09-21T10:47:33.841 INFO:teuthology.orchestra.run.smithi123.stdout: 5 244869 9.68 MB/sec execute 514 sec latency 50.641 ms 2024-09-21T10:47:34.087 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:34 smithi175 ceph-mon[49304]: Health check update: Degraded data redundancy: 175/1569 objects degraded (11.154%), 23 pgs degraded (PG_DEGRADED) 2024-09-21T10:47:34.201 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:34 smithi123 ceph-mon[63856]: Health check update: Degraded data redundancy: 175/1569 objects degraded (11.154%), 23 pgs degraded (PG_DEGRADED) 2024-09-21T10:47:34.841 INFO:teuthology.orchestra.run.smithi123.stdout: 5 245173 9.67 MB/sec execute 515 sec latency 61.575 ms 2024-09-21T10:47:35.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:35 smithi175 ceph-mon[49304]: pgmap v249: 97 pgs: 7 active+undersized, 23 active+undersized+degraded, 67 active+clean; 351 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.4 MiB/s wr, 475 op/s; 179/1626 objects degraded (11.009%) 2024-09-21T10:47:35.241 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:35 smithi123 ceph-mon[63856]: pgmap v249: 97 pgs: 7 active+undersized, 23 active+undersized+degraded, 67 active+clean; 351 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.4 MiB/s wr, 475 op/s; 179/1626 objects degraded (11.009%) 2024-09-21T10:47:35.841 INFO:teuthology.orchestra.run.smithi123.stdout: 5 245463 9.67 MB/sec execute 516 sec latency 51.881 ms 2024-09-21T10:47:36.841 INFO:teuthology.orchestra.run.smithi123.stdout: 5 245663 9.66 MB/sec execute 517 sec latency 59.673 ms 2024-09-21T10:47:37.535 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:37 smithi123 ceph-mon[63856]: pgmap v250: 97 pgs: 7 active+undersized, 23 active+undersized+degraded, 67 active+clean; 353 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 4.2 MiB/s wr, 383 op/s; 184/1668 objects degraded (11.031%) 2024-09-21T10:47:37.536 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:37 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:47:37.536 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:37 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:47:37.536 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:37 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:47:37.536 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:37 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:47:37.536 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:37 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:47:37.536 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:37 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:47:37.536 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:37 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:47:37.536 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:37 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:47:37.537 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:37 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:47:37.537 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:37 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:47:37.537 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:37 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-21T10:47:37.623 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:47:37 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-5[55349]: 2024-09-21T10:47:37.239+0000 7f2561123640 -1 osd.5 0 read_superblock omap replica is missing. 2024-09-21T10:47:37.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:37 smithi175 ceph-mon[49304]: pgmap v250: 97 pgs: 7 active+undersized, 23 active+undersized+degraded, 67 active+clean; 353 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 4.2 MiB/s wr, 383 op/s; 184/1668 objects degraded (11.031%) 2024-09-21T10:47:37.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:37 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:47:37.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:37 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:47:37.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:37 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:47:37.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:37 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:47:37.625 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:37 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:47:37.625 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:37 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:47:37.625 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:37 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:47:37.625 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:37 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:47:37.625 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:37 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:47:37.625 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:37 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:47:37.625 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:37 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-21T10:47:37.841 INFO:teuthology.orchestra.run.smithi123.stdout: 5 245966 9.66 MB/sec execute 518 sec latency 58.157 ms 2024-09-21T10:47:38.557 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:38 smithi123 ceph-mon[63856]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-21T10:47:38.557 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:38 smithi123 ceph-mon[63856]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-21T10:47:38.841 INFO:teuthology.orchestra.run.smithi123.stdout: 5 246306 9.64 MB/sec execute 519 sec latency 60.911 ms 2024-09-21T10:47:38.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:38 smithi175 ceph-mon[49304]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-21T10:47:38.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:38 smithi175 ceph-mon[49304]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-21T10:47:38.873 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:47:38 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-5[55349]: 2024-09-21T10:47:38.429+0000 7f2561123640 -1 osd.5 144 log_to_monitors true 2024-09-21T10:47:39.586 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:39 smithi123 ceph-mon[63856]: pgmap v251: 97 pgs: 7 active+undersized, 23 active+undersized+degraded, 67 active+clean; 353 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.6 MiB/s wr, 356 op/s; 186/1668 objects degraded (11.151%) 2024-09-21T10:47:39.586 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:39 smithi123 ceph-mon[63856]: Health check update: Degraded data redundancy: 186/1668 objects degraded (11.151%), 23 pgs degraded (PG_DEGRADED) 2024-09-21T10:47:39.586 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:39 smithi123 ceph-mon[63856]: from='osd.5 [v2:172.21.15.175:6808/2255404736,v1:172.21.15.175:6809/2255404736]' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-21T10:47:39.586 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:39 smithi123 ceph-mon[63856]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-21T10:47:39.841 INFO:teuthology.orchestra.run.smithi123.stdout: 5 246599 9.63 MB/sec execute 520 sec latency 43.059 ms 2024-09-21T10:47:39.873 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:47:39 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-5[55349]: 2024-09-21T10:47:39.481+0000 7f2558ecd640 -1 osd.5 144 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T10:47:39.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:39 smithi175 ceph-mon[49304]: pgmap v251: 97 pgs: 7 active+undersized, 23 active+undersized+degraded, 67 active+clean; 353 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.6 MiB/s wr, 356 op/s; 186/1668 objects degraded (11.151%) 2024-09-21T10:47:39.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:39 smithi175 ceph-mon[49304]: Health check update: Degraded data redundancy: 186/1668 objects degraded (11.151%), 23 pgs degraded (PG_DEGRADED) 2024-09-21T10:47:39.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:39 smithi175 ceph-mon[49304]: from='osd.5 [v2:172.21.15.175:6808/2255404736,v1:172.21.15.175:6809/2255404736]' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-21T10:47:39.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:39 smithi175 ceph-mon[49304]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-21T10:47:40.537 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:40 smithi123 ceph-mon[63856]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-21T10:47:40.537 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:40 smithi123 ceph-mon[63856]: osdmap e147: 8 total, 7 up, 8 in 2024-09-21T10:47:40.537 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:40 smithi123 ceph-mon[63856]: from='osd.5 [v2:172.21.15.175:6808/2255404736,v1:172.21.15.175:6809/2255404736]' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi175", "root=default"]} : dispatch 2024-09-21T10:47:40.538 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:40 smithi123 ceph-mon[63856]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi175", "root=default"]} : dispatch 2024-09-21T10:47:40.841 INFO:teuthology.orchestra.run.smithi123.stdout: 5 246825 9.63 MB/sec execute 521 sec latency 316.231 ms 2024-09-21T10:47:40.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:40 smithi175 ceph-mon[49304]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-21T10:47:40.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:40 smithi175 ceph-mon[49304]: osdmap e147: 8 total, 7 up, 8 in 2024-09-21T10:47:40.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:40 smithi175 ceph-mon[49304]: from='osd.5 [v2:172.21.15.175:6808/2255404736,v1:172.21.15.175:6809/2255404736]' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi175", "root=default"]} : dispatch 2024-09-21T10:47:40.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:40 smithi175 ceph-mon[49304]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi175", "root=default"]} : dispatch 2024-09-21T10:47:41.842 INFO:teuthology.orchestra.run.smithi123.stdout: 5 246872 9.61 MB/sec execute 522 sec latency 1316.295 ms 2024-09-21T10:47:41.850 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:41 smithi123 ceph-mon[63856]: pgmap v253: 97 pgs: 7 active+undersized, 23 active+undersized+degraded, 67 active+clean; 352 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.2 MiB/s wr, 406 op/s; 179/1620 objects degraded (11.049%) 2024-09-21T10:47:41.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:41 smithi123 ceph-mon[63856]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T10:47:41.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:41 smithi123 ceph-mon[63856]: osd.5 [v2:172.21.15.175:6808/2255404736,v1:172.21.15.175:6809/2255404736] boot 2024-09-21T10:47:41.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:41 smithi123 ceph-mon[63856]: osdmap e148: 8 total, 8 up, 8 in 2024-09-21T10:47:41.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:41 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-21T10:47:41.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:41 smithi175 ceph-mon[49304]: pgmap v253: 97 pgs: 7 active+undersized, 23 active+undersized+degraded, 67 active+clean; 352 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.2 MiB/s wr, 406 op/s; 179/1620 objects degraded (11.049%) 2024-09-21T10:47:41.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:41 smithi175 ceph-mon[49304]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T10:47:41.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:41 smithi175 ceph-mon[49304]: osd.5 [v2:172.21.15.175:6808/2255404736,v1:172.21.15.175:6809/2255404736] boot 2024-09-21T10:47:41.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:41 smithi175 ceph-mon[49304]: osdmap e148: 8 total, 8 up, 8 in 2024-09-21T10:47:41.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:41 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-21T10:47:42.842 INFO:teuthology.orchestra.run.smithi123.stdout: 5 246872 9.60 MB/sec execute 523 sec latency 2316.330 ms 2024-09-21T10:47:42.847 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:42 smithi123 ceph-mon[63856]: osdmap e149: 8 total, 8 up, 8 in 2024-09-21T10:47:42.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:42 smithi175 ceph-mon[49304]: osdmap e149: 8 total, 8 up, 8 in 2024-09-21T10:47:43.627 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:43 smithi123 ceph-mon[63856]: pgmap v256: 97 pgs: 1 peering, 6 active+undersized, 23 active+undersized+degraded, 67 active+clean; 348 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.4 MiB/s wr, 360 op/s; 181/1608 objects degraded (11.256%) 2024-09-21T10:47:43.627 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:43 smithi123 ceph-mon[63856]: osdmap e150: 8 total, 8 up, 8 in 2024-09-21T10:47:43.627 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:43 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:47:43.627 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:43 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:47:43.774 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:43 smithi175 ceph-mon[49304]: pgmap v256: 97 pgs: 1 peering, 6 active+undersized, 23 active+undersized+degraded, 67 active+clean; 348 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.4 MiB/s wr, 360 op/s; 181/1608 objects degraded (11.256%) 2024-09-21T10:47:43.774 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:43 smithi175 ceph-mon[49304]: osdmap e150: 8 total, 8 up, 8 in 2024-09-21T10:47:43.774 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:43 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:47:43.774 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:43 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:47:43.841 INFO:teuthology.orchestra.run.smithi123.stdout: 5 247216 9.61 MB/sec execute 524 sec latency 2334.569 ms 2024-09-21T10:47:44.842 INFO:teuthology.orchestra.run.smithi123.stdout: 5 247374 9.60 MB/sec execute 525 sec latency 987.217 ms 2024-09-21T10:47:44.892 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:44 smithi123 ceph-mon[63856]: osdmap e151: 8 total, 8 up, 8 in 2024-09-21T10:47:45.016 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:44 smithi175 ceph-mon[49304]: osdmap e151: 8 total, 8 up, 8 in 2024-09-21T10:47:45.673 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:45 smithi123 ceph-mon[63856]: pgmap v259: 97 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 1 peering, 2 active+recovering+undersized+degraded+remapped, 3 active+undersized, 11 active+undersized+degraded, 76 active+clean; 345 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.2 MiB/s wr, 367 op/s; 135/1557 objects degraded (8.671%); 2.0 MiB/s, 23 objects/s recovering 2024-09-21T10:47:45.673 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:45 smithi123 ceph-mon[63856]: Health check update: Degraded data redundancy: 135/1557 objects degraded (8.671%), 15 pgs degraded (PG_DEGRADED) 2024-09-21T10:47:45.673 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:45 smithi123 ceph-mon[63856]: osdmap e152: 8 total, 8 up, 8 in 2024-09-21T10:47:45.842 INFO:teuthology.orchestra.run.smithi123.stdout: 5 247712 9.59 MB/sec execute 526 sec latency 686.614 ms 2024-09-21T10:47:46.027 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:45 smithi175 ceph-mon[49304]: pgmap v259: 97 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 1 peering, 2 active+recovering+undersized+degraded+remapped, 3 active+undersized, 11 active+undersized+degraded, 76 active+clean; 345 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.2 MiB/s wr, 367 op/s; 135/1557 objects degraded (8.671%); 2.0 MiB/s, 23 objects/s recovering 2024-09-21T10:47:46.028 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:45 smithi175 ceph-mon[49304]: Health check update: Degraded data redundancy: 135/1557 objects degraded (8.671%), 15 pgs degraded (PG_DEGRADED) 2024-09-21T10:47:46.028 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:45 smithi175 ceph-mon[49304]: osdmap e152: 8 total, 8 up, 8 in 2024-09-21T10:47:46.684 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:46 smithi123 ceph-mon[63856]: osdmap e153: 8 total, 8 up, 8 in 2024-09-21T10:47:46.842 INFO:teuthology.orchestra.run.smithi123.stdout: 5 248044 9.59 MB/sec execute 527 sec latency 55.800 ms 2024-09-21T10:47:47.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:46 smithi175 ceph-mon[49304]: osdmap e153: 8 total, 8 up, 8 in 2024-09-21T10:47:47.721 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:47 smithi123 ceph-mon[63856]: pgmap v262: 97 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 4 peering, 89 active+clean; 346 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.7 MiB/s wr, 359 op/s; 43/1560 objects degraded (2.756%); 2.3 MiB/s, 34 objects/s recovering 2024-09-21T10:47:47.842 INFO:teuthology.orchestra.run.smithi123.stdout: 5 248222 9.58 MB/sec execute 528 sec latency 40.501 ms 2024-09-21T10:47:48.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:47 smithi175 ceph-mon[49304]: pgmap v262: 97 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 4 peering, 89 active+clean; 346 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.7 MiB/s wr, 359 op/s; 43/1560 objects degraded (2.756%); 2.3 MiB/s, 34 objects/s recovering 2024-09-21T10:47:48.842 INFO:teuthology.orchestra.run.smithi123.stdout: 5 248479 9.58 MB/sec execute 529 sec latency 55.851 ms 2024-09-21T10:47:49.788 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:49 smithi123 ceph-mon[63856]: pgmap v263: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 4 peering, 91 active+clean; 347 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 2.9 MiB/s wr, 265 op/s; 20/1599 objects degraded (1.251%); 1.6 MiB/s, 28 objects/s recovering 2024-09-21T10:47:49.842 INFO:teuthology.orchestra.run.smithi123.stdout: 5 248905 9.59 MB/sec execute 530 sec latency 59.943 ms 2024-09-21T10:47:50.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:49 smithi175 ceph-mon[49304]: pgmap v263: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 4 peering, 91 active+clean; 347 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 2.9 MiB/s wr, 265 op/s; 20/1599 objects degraded (1.251%); 1.6 MiB/s, 28 objects/s recovering 2024-09-21T10:47:50.832 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:50 smithi123 ceph-mon[63856]: Health check update: Degraded data redundancy: 20/1599 objects degraded (1.251%), 1 pg degraded (PG_DEGRADED) 2024-09-21T10:47:50.842 INFO:teuthology.orchestra.run.smithi123.stdout: 5 249361 9.59 MB/sec execute 531 sec latency 44.246 ms 2024-09-21T10:47:51.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:50 smithi175 ceph-mon[49304]: Health check update: Degraded data redundancy: 20/1599 objects degraded (1.251%), 1 pg degraded (PG_DEGRADED) 2024-09-21T10:47:51.842 INFO:teuthology.orchestra.run.smithi123.stdout: 5 249826 9.59 MB/sec execute 532 sec latency 36.516 ms 2024-09-21T10:47:51.843 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:51 smithi123 ceph-mon[63856]: pgmap v264: 97 pgs: 97 active+clean; 354 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 3.2 KiB/s rd, 3.4 MiB/s wr, 293 op/s; 292 KiB/s, 24 objects/s recovering 2024-09-21T10:47:51.843 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:51 smithi123 ceph-mon[63856]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 20/1599 objects degraded (1.251%), 1 pg degraded) 2024-09-21T10:47:51.843 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:51 smithi123 ceph-mon[63856]: Cluster is now healthy 2024-09-21T10:47:52.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:51 smithi175 ceph-mon[49304]: pgmap v264: 97 pgs: 97 active+clean; 354 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 3.2 KiB/s rd, 3.4 MiB/s wr, 293 op/s; 292 KiB/s, 24 objects/s recovering 2024-09-21T10:47:52.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:51 smithi175 ceph-mon[49304]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 20/1599 objects degraded (1.251%), 1 pg degraded) 2024-09-21T10:47:52.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:51 smithi175 ceph-mon[49304]: Cluster is now healthy 2024-09-21T10:47:52.842 INFO:teuthology.orchestra.run.smithi123.stdout: 5 250234 9.59 MB/sec execute 533 sec latency 35.359 ms 2024-09-21T10:47:52.862 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:52 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-21T10:47:52.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:52 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-21T10:47:53.842 INFO:teuthology.orchestra.run.smithi123.stdout: 5 250568 9.59 MB/sec execute 534 sec latency 52.281 ms 2024-09-21T10:47:53.876 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:53 smithi123 ceph-mon[63856]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-21T10:47:53.876 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:53 smithi123 ceph-mon[63856]: Upgrade: osd.6 is safe to restart 2024-09-21T10:47:53.876 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:53 smithi123 ceph-mon[63856]: pgmap v265: 97 pgs: 97 active+clean; 356 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 2.6 KiB/s rd, 3.6 MiB/s wr, 276 op/s; 243 KiB/s, 20 objects/s recovering 2024-09-21T10:47:53.877 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:53 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:47:53.877 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:53 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-21T10:47:53.877 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:53 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:47:53.975 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:53 smithi175 ceph-mon[49304]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-21T10:47:53.975 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:53 smithi175 ceph-mon[49304]: Upgrade: osd.6 is safe to restart 2024-09-21T10:47:53.975 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:53 smithi175 ceph-mon[49304]: pgmap v265: 97 pgs: 97 active+clean; 356 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 2.6 KiB/s rd, 3.6 MiB/s wr, 276 op/s; 243 KiB/s, 20 objects/s recovering 2024-09-21T10:47:53.976 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:53 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:47:53.976 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:53 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-21T10:47:53.976 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:53 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:47:54.842 INFO:teuthology.orchestra.run.smithi123.stdout: 5 250955 9.60 MB/sec execute 535 sec latency 42.395 ms 2024-09-21T10:47:54.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:54 smithi175 ceph-mon[49304]: Upgrade: Updating osd.6 2024-09-21T10:47:54.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:54 smithi175 ceph-mon[49304]: Deploying daemon osd.6 on smithi175 2024-09-21T10:47:54.898 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:54 smithi123 ceph-mon[63856]: Upgrade: Updating osd.6 2024-09-21T10:47:54.898 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:54 smithi123 ceph-mon[63856]: Deploying daemon osd.6 on smithi175 2024-09-21T10:47:55.842 INFO:teuthology.orchestra.run.smithi123.stdout: 5 251383 9.60 MB/sec execute 536 sec latency 38.191 ms 2024-09-21T10:47:55.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:55 smithi175 ceph-mon[49304]: pgmap v266: 97 pgs: 97 active+clean; 356 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.6 MiB/s wr, 463 op/s; 168 KiB/s, 19 objects/s recovering 2024-09-21T10:47:55.922 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:55 smithi123 ceph-mon[63856]: pgmap v266: 97 pgs: 97 active+clean; 356 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.6 MiB/s wr, 463 op/s; 168 KiB/s, 19 objects/s recovering 2024-09-21T10:47:56.842 INFO:teuthology.orchestra.run.smithi123.stdout: 5 251660 9.60 MB/sec execute 537 sec latency 42.387 ms 2024-09-21T10:47:57.024 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:47:56 smithi175 systemd[1]: Stopping Ceph osd.6 for 3ccd15e6-7804-11ef-baf6-efdc52797490... 2024-09-21T10:47:57.373 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:47:57 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6[35907]: 2024-09-21T10:47:57.020+0000 7f92d6f02700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:47:57.373 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:47:57 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6[35907]: 2024-09-21T10:47:57.020+0000 7f92d6f02700 -1 osd.6 153 *** Got signal Terminated *** 2024-09-21T10:47:57.373 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:47:57 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6[35907]: 2024-09-21T10:47:57.020+0000 7f92d6f02700 -1 osd.6 153 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T10:47:57.721 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:57 smithi123 ceph-mon[63856]: pgmap v267: 97 pgs: 97 active+clean; 351 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.9 MiB/s wr, 464 op/s; 75 KiB/s, 10 objects/s recovering 2024-09-21T10:47:57.721 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:57 smithi123 ceph-mon[63856]: osd.6 marked itself down and dead 2024-09-21T10:47:57.721 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:57 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:47:57.721 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:57 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:47:57.842 INFO:teuthology.orchestra.run.smithi123.stdout: 5 251908 9.59 MB/sec execute 538 sec latency 111.716 ms 2024-09-21T10:47:57.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:57 smithi175 ceph-mon[49304]: pgmap v267: 97 pgs: 97 active+clean; 351 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.9 MiB/s wr, 464 op/s; 75 KiB/s, 10 objects/s recovering 2024-09-21T10:47:57.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:57 smithi175 ceph-mon[49304]: osd.6 marked itself down and dead 2024-09-21T10:47:57.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:57 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:47:57.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:57 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:47:58.623 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:47:58 smithi175 podman[56909]: 2024-09-21 10:47:58.34406052 +0000 UTC m=+1.448774989 container died 9bcbf018eb20b2a83543dc19153d18e524d9576d4f695ad552a96b0859f55562 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6, ceph=True, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, GIT_CLEAN=True, io.openshift.expose-services=, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, vendor=Red Hat, Inc., version=8, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64) 2024-09-21T10:47:58.842 INFO:teuthology.orchestra.run.smithi123.stdout: 5 252023 9.57 MB/sec execute 539 sec latency 991.047 ms 2024-09-21T10:47:58.927 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:58 smithi123 ceph-mon[63856]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T10:47:58.927 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:58 smithi123 ceph-mon[63856]: osdmap e154: 8 total, 7 up, 8 in 2024-09-21T10:47:58.928 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:58 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:47:58.992 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:58 smithi175 ceph-mon[49304]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T10:47:58.992 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:58 smithi175 ceph-mon[49304]: osdmap e154: 8 total, 7 up, 8 in 2024-09-21T10:47:58.992 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:58 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:47:58.993 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:47:58 smithi175 podman[56909]: 2024-09-21 10:47:58.755360071 +0000 UTC m=+1.860074539 container cleanup 9bcbf018eb20b2a83543dc19153d18e524d9576d4f695ad552a96b0859f55562 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6, io.openshift.expose-services=, version=8, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, io.buildah.version=1.19.8, distribution-scope=public, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, RELEASE=HEAD, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, vcs-type=git) 2024-09-21T10:47:58.993 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:47:58 smithi175 bash[56909]: ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6 2024-09-21T10:47:59.346 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:47:58 smithi175 podman[56928]: 2024-09-21 10:47:58.990487958 +0000 UTC m=+0.640231238 container remove 9bcbf018eb20b2a83543dc19153d18e524d9576d4f695ad552a96b0859f55562 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6, ceph=True, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, architecture=x86_64, vendor=Red Hat, Inc., version=8, release=754, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True) 2024-09-21T10:47:59.626 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:47:59 smithi175 podman[57015]: 2024-09-21 10:47:59.247817254 +0000 UTC m=+0.018125805 image pull 8f4d013db793795fef09ff5edf96e6b42ba8cd2c9158b2f0b7a5d7c1e657c063 quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472 2024-09-21T10:47:59.626 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:47:59 smithi175 podman[57015]: 2024-09-21 10:47:59.374873939 +0000 UTC m=+0.145182472 container create d79426e361aa19617e37306ba9cdb1ef8bd67f18b0cfc59c153794fab4657808 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-09-21T10:47:59.842 INFO:teuthology.orchestra.run.smithi123.stdout: 5 252321 9.57 MB/sec execute 540 sec latency 145.780 ms 2024-09-21T10:47:59.948 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:59 smithi123 ceph-mon[63856]: pgmap v269: 97 pgs: 4 peering, 9 stale+active+clean, 84 active+clean; 353 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 5.2 MiB/s wr, 551 op/s; 77 KiB/s, 9 objects/s recovering 2024-09-21T10:47:59.948 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:59 smithi123 ceph-mon[63856]: osdmap e155: 8 total, 7 up, 8 in 2024-09-21T10:47:59.948 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:47:59 smithi123 ceph-mon[63856]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-21T10:48:00.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:59 smithi175 ceph-mon[49304]: pgmap v269: 97 pgs: 4 peering, 9 stale+active+clean, 84 active+clean; 353 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 5.2 MiB/s wr, 551 op/s; 77 KiB/s, 9 objects/s recovering 2024-09-21T10:48:00.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:59 smithi175 ceph-mon[49304]: osdmap e155: 8 total, 7 up, 8 in 2024-09-21T10:48:00.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:47:59 smithi175 ceph-mon[49304]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-21T10:48:00.124 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:47:59 smithi175 podman[57015]: 2024-09-21 10:47:59.681166637 +0000 UTC m=+0.451475188 container init d79426e361aa19617e37306ba9cdb1ef8bd67f18b0cfc59c153794fab4657808 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6-deactivate, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True) 2024-09-21T10:48:00.124 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:47:59 smithi175 podman[57015]: 2024-09-21 10:47:59.684647232 +0000 UTC m=+0.454955772 container start d79426e361aa19617e37306ba9cdb1ef8bd67f18b0cfc59c153794fab4657808 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6-deactivate, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T10:48:00.124 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:47:59 smithi175 podman[57015]: 2024-09-21 10:47:59.778202439 +0000 UTC m=+0.548510972 container attach d79426e361aa19617e37306ba9cdb1ef8bd67f18b0cfc59c153794fab4657808 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6-deactivate, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-09-21T10:48:00.124 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:47:59 smithi175 podman[57015]: 2024-09-21 10:47:59.883037085 +0000 UTC m=+0.653345629 container died d79426e361aa19617e37306ba9cdb1ef8bd67f18b0cfc59c153794fab4657808 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6-deactivate, CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_BRANCH=HEAD) 2024-09-21T10:48:00.320 INFO:teuthology.orchestra.run.smithi123.stdout:true 2024-09-21T10:48:00.842 INFO:teuthology.orchestra.run.smithi123.stdout: 5 252633 9.58 MB/sec execute 541 sec latency 59.942 ms 2024-09-21T10:48:00.988 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:00 smithi123 ceph-mon[63856]: Health check failed: Degraded data redundancy: 55/1653 objects degraded (3.327%), 6 pgs degraded (PG_DEGRADED) 2024-09-21T10:48:01.093 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:00 smithi175 ceph-mon[49304]: Health check failed: Degraded data redundancy: 55/1653 objects degraded (3.327%), 6 pgs degraded (PG_DEGRADED) 2024-09-21T10:48:01.093 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:48:00 smithi175 podman[57038]: 2024-09-21 10:48:00.633862422 +0000 UTC m=+0.742660334 container remove d79426e361aa19617e37306ba9cdb1ef8bd67f18b0cfc59c153794fab4657808 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6-deactivate, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-09-21T10:48:01.093 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:48:00 smithi175 systemd[1]: ceph-3ccd15e6-7804-11ef-baf6-efdc52797490@osd.6.service: Deactivated successfully. 2024-09-21T10:48:01.093 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:48:00 smithi175 systemd[1]: Stopped Ceph osd.6 for 3ccd15e6-7804-11ef-baf6-efdc52797490. 2024-09-21T10:48:01.093 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:48:00 smithi175 systemd[1]: ceph-3ccd15e6-7804-11ef-baf6-efdc52797490@osd.6.service: Consumed 4min 31.644s CPU time. 2024-09-21T10:48:01.093 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:48:00 smithi175 systemd[1]: Starting Ceph osd.6 for 3ccd15e6-7804-11ef-baf6-efdc52797490... 2024-09-21T10:48:01.093 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:48:01 smithi175 podman[57134]: 2024-09-21 10:48:00.993693968 +0000 UTC m=+0.017007335 image pull 8f4d013db793795fef09ff5edf96e6b42ba8cd2c9158b2f0b7a5d7c1e657c063 quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472 2024-09-21T10:48:01.374 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:48:01 smithi175 podman[57134]: 2024-09-21 10:48:01.130156589 +0000 UTC m=+0.153469960 container create 3417547a8fb8ac554b94959d935aa2048885b875356df8306cf2091d69e9e4ef (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6-activate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS) 2024-09-21T10:48:01.418 INFO:teuthology.orchestra.run.smithi123.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T10:48:01.418 INFO:teuthology.orchestra.run.smithi123.stdout:alertmanager.smithi123 smithi123 *:9093,9094 running (8m) 109s ago 17m 22.0M - 0.27.0 11f11916f8cd 5d550bd97de5 2024-09-21T10:48:01.418 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi123 smithi123 running (5m) 109s ago 17m 6899k - 19.3.0-5103-g5c1cdc3c 8f4d013db793 c5e1ef39a0a8 2024-09-21T10:48:01.418 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi175 smithi175 running (5m) 30s ago 17m 6895k - 19.3.0-5103-g5c1cdc3c 8f4d013db793 fb1ca999ccfd 2024-09-21T10:48:01.419 INFO:teuthology.orchestra.run.smithi123.stdout:grafana.smithi123 smithi123 *:3000 running (8m) 109s ago 17m 52.6M - dad864ee21e9 771df81763ba 2024-09-21T10:48:01.419 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi123.uqoyow smithi123 running (11m) 109s ago 11m 41.2M - 17.2.0 e1d6a67b021e f0a21eaac266 2024-09-21T10:48:01.419 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi175.jrlevw smithi175 running (11m) 30s ago 11m 14.4M - 17.2.0 e1d6a67b021e b21b16f4bc50 2024-09-21T10:48:01.419 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi123.stnuit smithi123 *:8443,9283,8765 running (7m) 109s ago 18m 519M - 19.3.0-5103-g5c1cdc3c 8f4d013db793 6812cc5cb56b 2024-09-21T10:48:01.419 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi175.txvwrq smithi175 *:8443,9283,8765 running (6m) 30s ago 16m 456M - 19.3.0-5103-g5c1cdc3c 8f4d013db793 304f0408075e 2024-09-21T10:48:01.419 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi123 smithi123 running (6m) 109s ago 19m 52.0M 2048M 19.3.0-5103-g5c1cdc3c 8f4d013db793 6b95207a2d80 2024-09-21T10:48:01.419 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi175 smithi175 running (6m) 30s ago 16m 45.0M 2048M 19.3.0-5103-g5c1cdc3c 8f4d013db793 3abee2d37f0f 2024-09-21T10:48:01.419 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.0.0.smithi123.ddatpr smithi123 *:2049 running (11m) 109s ago 11m 281M - 4.0 e1d6a67b021e 36f8cb2281eb 2024-09-21T10:48:01.419 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.1.0.smithi175.wpawer smithi175 *:2049 running (11m) 30s ago 11m 90.8M - 4.0 e1d6a67b021e 2200aeed9200 2024-09-21T10:48:01.419 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi123 smithi123 *:9100 running (8m) 109s ago 17m 12.3M - 1.7.0 72c9c2088986 ca7d0b29af81 2024-09-21T10:48:01.419 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi175 smithi175 *:9100 running (7m) 30s ago 16m 10.8M - 1.7.0 72c9c2088986 46c4f6b7c149 2024-09-21T10:48:01.419 INFO:teuthology.orchestra.run.smithi123.stdout:osd.0 smithi123 running (4m) 109s ago 16m 362M 1936M 19.3.0-5103-g5c1cdc3c 8f4d013db793 cc40f695c3a8 2024-09-21T10:48:01.419 INFO:teuthology.orchestra.run.smithi123.stdout:osd.1 smithi123 running (3m) 109s ago 15m 279M 1936M 19.3.0-5103-g5c1cdc3c 8f4d013db793 1be249025c7f 2024-09-21T10:48:01.420 INFO:teuthology.orchestra.run.smithi123.stdout:osd.2 smithi123 running (2m) 109s ago 15m 232M 1936M 19.3.0-5103-g5c1cdc3c 8f4d013db793 647aec759e3c 2024-09-21T10:48:01.420 INFO:teuthology.orchestra.run.smithi123.stdout:osd.3 smithi123 running (118s) 109s ago 14m 11.6M 1936M 19.3.0-5103-g5c1cdc3c 8f4d013db793 7a082fcad8e8 2024-09-21T10:48:01.420 INFO:teuthology.orchestra.run.smithi123.stdout:osd.4 smithi175 running (72s) 30s ago 14m 261M 2704M 19.3.0-5103-g5c1cdc3c 8f4d013db793 b650f72d10f0 2024-09-21T10:48:01.420 INFO:teuthology.orchestra.run.smithi123.stdout:osd.5 smithi175 running (35s) 30s ago 13m 14.3M 2704M 19.3.0-5103-g5c1cdc3c 8f4d013db793 8199a0b362df 2024-09-21T10:48:01.420 INFO:teuthology.orchestra.run.smithi123.stdout:osd.6 smithi175 running (13m) 30s ago 13m 509M 2704M 17.2.0 e1d6a67b021e 9bcbf018eb20 2024-09-21T10:48:01.420 INFO:teuthology.orchestra.run.smithi123.stdout:osd.7 smithi175 running (13m) 30s ago 13m 768M 2704M 17.2.0 e1d6a67b021e e9b8f1f90994 2024-09-21T10:48:01.420 INFO:teuthology.orchestra.run.smithi123.stdout:prometheus.smithi123 smithi123 *:9095 running (6m) 109s ago 16m 55.4M - 2.51.0 1d3b7f56885b 37f944c9de88 2024-09-21T10:48:01.753 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:48:01 smithi175 podman[57134]: 2024-09-21 10:48:01.430634038 +0000 UTC m=+0.453947412 container init 3417547a8fb8ac554b94959d935aa2048885b875356df8306cf2091d69e9e4ef (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6-activate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, ceph=True) 2024-09-21T10:48:01.753 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:48:01 smithi175 podman[57134]: 2024-09-21 10:48:01.435067582 +0000 UTC m=+0.458380952 container start 3417547a8fb8ac554b94959d935aa2048885b875356df8306cf2091d69e9e4ef (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6-activate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS) 2024-09-21T10:48:01.753 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:48:01 smithi175 podman[57134]: 2024-09-21 10:48:01.522369979 +0000 UTC m=+0.545683362 container attach 3417547a8fb8ac554b94959d935aa2048885b875356df8306cf2091d69e9e4ef (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6-activate, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux ) 2024-09-21T10:48:01.753 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:48:01 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6-activate[57149]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:48:01.753 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:48:01 smithi175 bash[57134]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:48:01.753 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:48:01 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6-activate[57149]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:48:01.753 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:48:01 smithi175 bash[57134]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:48:01.842 INFO:teuthology.orchestra.run.smithi123.stdout: 5 252990 9.58 MB/sec execute 542 sec latency 56.751 ms 2024-09-21T10:48:02.003 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:01 smithi123 ceph-mon[63856]: pgmap v271: 97 pgs: 2 active+undersized, 9 peering, 6 active+undersized+degraded, 80 active+clean; 350 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 5.8 MiB/s wr, 641 op/s; 55/1653 objects degraded (3.327%) 2024-09-21T10:48:02.003 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:01 smithi123 ceph-mon[63856]: from='client.34410 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:48:02.021 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-09-21T10:48:02.021 INFO:teuthology.orchestra.run.smithi123.stdout: "mon": { 2024-09-21T10:48:02.021 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev)": 2 2024-09-21T10:48:02.021 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:48:02.022 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr": { 2024-09-21T10:48:02.022 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev)": 2 2024-09-21T10:48:02.022 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:48:02.022 INFO:teuthology.orchestra.run.smithi123.stdout: "osd": { 2024-09-21T10:48:02.022 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-21T10:48:02.022 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev)": 6 2024-09-21T10:48:02.022 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:48:02.022 INFO:teuthology.orchestra.run.smithi123.stdout: "mds": { 2024-09-21T10:48:02.022 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T10:48:02.022 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:48:02.022 INFO:teuthology.orchestra.run.smithi123.stdout: "overall": { 2024-09-21T10:48:02.022 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-09-21T10:48:02.022 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev)": 10 2024-09-21T10:48:02.023 INFO:teuthology.orchestra.run.smithi123.stdout: } 2024-09-21T10:48:02.023 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-09-21T10:48:02.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:01 smithi175 ceph-mon[49304]: pgmap v271: 97 pgs: 2 active+undersized, 9 peering, 6 active+undersized+degraded, 80 active+clean; 350 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 5.8 MiB/s wr, 641 op/s; 55/1653 objects degraded (3.327%) 2024-09-21T10:48:02.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:01 smithi175 ceph-mon[49304]: from='client.34410 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:48:02.523 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-09-21T10:48:02.523 INFO:teuthology.orchestra.run.smithi123.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472", 2024-09-21T10:48:02.523 INFO:teuthology.orchestra.run.smithi123.stdout: "in_progress": true, 2024-09-21T10:48:02.523 INFO:teuthology.orchestra.run.smithi123.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T10:48:02.523 INFO:teuthology.orchestra.run.smithi123.stdout: "services_complete": [ 2024-09-21T10:48:02.523 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr", 2024-09-21T10:48:02.523 INFO:teuthology.orchestra.run.smithi123.stdout: "crash", 2024-09-21T10:48:02.523 INFO:teuthology.orchestra.run.smithi123.stdout: "mon" 2024-09-21T10:48:02.523 INFO:teuthology.orchestra.run.smithi123.stdout: ], 2024-09-21T10:48:02.523 INFO:teuthology.orchestra.run.smithi123.stdout: "progress": "12/23 daemons upgraded", 2024-09-21T10:48:02.524 INFO:teuthology.orchestra.run.smithi123.stdout: "message": "Currently upgrading osd daemons", 2024-09-21T10:48:02.524 INFO:teuthology.orchestra.run.smithi123.stdout: "is_paused": false 2024-09-21T10:48:02.524 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-09-21T10:48:02.770 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:02 smithi123 ceph-mon[63856]: from='client.34414 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:48:02.770 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:02 smithi123 ceph-mon[63856]: from='client.34418 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:48:02.770 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:02 smithi123 ceph-mon[63856]: from='client.? 172.21.15.123:0/3977197258' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:48:02.842 INFO:teuthology.orchestra.run.smithi123.stdout: 5 253303 9.57 MB/sec execute 543 sec latency 80.294 ms 2024-09-21T10:48:03.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:02 smithi175 ceph-mon[49304]: from='client.34414 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:48:03.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:02 smithi175 ceph-mon[49304]: from='client.34418 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:48:03.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:02 smithi175 ceph-mon[49304]: from='client.? 172.21.15.123:0/3977197258' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:48:03.164 INFO:teuthology.orchestra.run.smithi123.stdout:HEALTH_WARN 1 osds down; Reduced data availability: 1 pg peering; Degraded data redundancy: 128/1623 objects degraded (7.887%), 17 pgs degraded 2024-09-21T10:48:03.164 INFO:teuthology.orchestra.run.smithi123.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-21T10:48:03.164 INFO:teuthology.orchestra.run.smithi123.stdout: osd.6 (root=default,host=smithi175) is down 2024-09-21T10:48:03.164 INFO:teuthology.orchestra.run.smithi123.stdout:[WRN] PG_AVAILABILITY: Reduced data availability: 1 pg peering 2024-09-21T10:48:03.164 INFO:teuthology.orchestra.run.smithi123.stdout: pg 4.a is stuck peering for 11m, current state peering, last acting [1,7] 2024-09-21T10:48:03.164 INFO:teuthology.orchestra.run.smithi123.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 128/1623 objects degraded (7.887%), 17 pgs degraded 2024-09-21T10:48:03.164 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.2 is active+undersized+degraded, acting [5,1] 2024-09-21T10:48:03.165 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.11 is active+undersized+degraded, acting [4,1] 2024-09-21T10:48:03.165 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.14 is active+undersized+degraded, acting [3,5] 2024-09-21T10:48:03.165 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.16 is active+undersized+degraded, acting [5,2] 2024-09-21T10:48:03.165 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.17 is active+undersized+degraded, acting [5,2] 2024-09-21T10:48:03.165 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.19 is active+undersized+degraded, acting [3,0] 2024-09-21T10:48:03.165 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.1a is active+undersized+degraded, acting [4,7] 2024-09-21T10:48:03.165 INFO:teuthology.orchestra.run.smithi123.stdout: pg 2.1b is active+undersized+degraded, acting [3,7] 2024-09-21T10:48:03.165 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.2 is active+undersized+degraded, acting [3,5] 2024-09-21T10:48:03.165 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.3 is active+undersized+degraded, acting [4,0] 2024-09-21T10:48:03.165 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.a is active+undersized+degraded, acting [4,1] 2024-09-21T10:48:03.165 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.c is active+undersized+degraded, acting [5,3] 2024-09-21T10:48:03.165 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.10 is active+undersized+degraded, acting [0,5] 2024-09-21T10:48:03.165 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.14 is active+undersized+degraded, acting [4,7] 2024-09-21T10:48:03.165 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.1d is active+undersized+degraded, acting [5,4] 2024-09-21T10:48:03.166 INFO:teuthology.orchestra.run.smithi123.stdout: pg 3.1e is active+undersized+degraded, acting [3,2] 2024-09-21T10:48:03.166 INFO:teuthology.orchestra.run.smithi123.stdout: pg 4.5 is active+undersized+degraded, acting [4,7] 2024-09-21T10:48:03.781 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:03 smithi123 ceph-mon[63856]: pgmap v272: 97 pgs: 9 active+undersized, 9 peering, 17 active+undersized+degraded, 62 active+clean; 353 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4.6 KiB/s rd, 4.7 MiB/s wr, 439 op/s; 128/1623 objects degraded (7.887%) 2024-09-21T10:48:03.781 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:03 smithi123 ceph-mon[63856]: from='client.34426 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:48:03.781 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:03 smithi123 ceph-mon[63856]: from='client.? 172.21.15.123:0/1869221890' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T10:48:03.842 INFO:teuthology.orchestra.run.smithi123.stdout: 5 253659 9.56 MB/sec execute 544 sec latency 40.979 ms 2024-09-21T10:48:04.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:03 smithi175 ceph-mon[49304]: pgmap v272: 97 pgs: 9 active+undersized, 9 peering, 17 active+undersized+degraded, 62 active+clean; 353 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4.6 KiB/s rd, 4.7 MiB/s wr, 439 op/s; 128/1623 objects degraded (7.887%) 2024-09-21T10:48:04.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:03 smithi175 ceph-mon[49304]: from='client.34426 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:48:04.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:03 smithi175 ceph-mon[49304]: from='client.? 172.21.15.123:0/1869221890' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T10:48:04.842 INFO:teuthology.orchestra.run.smithi123.stdout: 5 253945 9.56 MB/sec execute 545 sec latency 52.797 ms 2024-09-21T10:48:04.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:04 smithi175 ceph-mon[49304]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-21T10:48:04.873 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:48:04 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6-activate[57149]: --> Failed to activate via raw: did not find any matching OSD to activate 2024-09-21T10:48:04.874 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:48:04 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6-activate[57149]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:48:04.874 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:48:04 smithi175 bash[57134]: --> Failed to activate via raw: did not find any matching OSD to activate 2024-09-21T10:48:04.874 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:48:04 smithi175 bash[57134]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:48:04.874 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:48:04 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6-activate[57149]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:48:04.874 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:48:04 smithi175 bash[57134]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:48:04.874 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:48:04 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6-activate[57149]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-21T10:48:04.874 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:48:04 smithi175 bash[57134]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-21T10:48:04.874 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:48:04 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6-activate[57149]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-6 --no-mon-config 2024-09-21T10:48:04.874 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:48:04 smithi175 bash[57134]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-6 --no-mon-config 2024-09-21T10:48:05.059 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:04 smithi123 ceph-mon[63856]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-21T10:48:05.298 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:48:04 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6-activate[57149]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-6/block 2024-09-21T10:48:05.298 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:48:04 smithi175 bash[57134]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-6/block 2024-09-21T10:48:05.298 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:48:04 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6-activate[57149]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-6/block 2024-09-21T10:48:05.298 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:48:04 smithi175 bash[57134]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-6/block 2024-09-21T10:48:05.299 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:48:04 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6-activate[57149]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-21T10:48:05.299 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:48:04 smithi175 bash[57134]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-21T10:48:05.299 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:48:04 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6-activate[57149]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-21T10:48:05.299 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:48:04 smithi175 bash[57134]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-21T10:48:05.299 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:48:04 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6-activate[57149]: --> ceph-volume lvm activate successful for osd ID: 6 2024-09-21T10:48:05.299 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:48:04 smithi175 bash[57134]: --> ceph-volume lvm activate successful for osd ID: 6 2024-09-21T10:48:05.299 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:48:04 smithi175 podman[57134]: 2024-09-21 10:48:04.979847751 +0000 UTC m=+4.003161126 container died 3417547a8fb8ac554b94959d935aa2048885b875356df8306cf2091d69e9e4ef (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6-activate, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-09-21T10:48:05.825 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:05 smithi123 ceph-mon[63856]: pgmap v273: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 348 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4.4 KiB/s rd, 3.8 MiB/s wr, 388 op/s; 192/1617 objects degraded (11.874%) 2024-09-21T10:48:05.842 INFO:teuthology.orchestra.run.smithi123.stdout: 5 254298 9.57 MB/sec execute 546 sec latency 44.572 ms 2024-09-21T10:48:05.937 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:05 smithi175 ceph-mon[49304]: pgmap v273: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 348 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4.4 KiB/s rd, 3.8 MiB/s wr, 388 op/s; 192/1617 objects degraded (11.874%) 2024-09-21T10:48:05.937 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:48:05 smithi175 podman[57353]: 2024-09-21 10:48:05.682848014 +0000 UTC m=+0.695562050 container remove 3417547a8fb8ac554b94959d935aa2048885b875356df8306cf2091d69e9e4ef (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6-activate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-09-21T10:48:05.937 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:48:05 smithi175 podman[57414]: 2024-09-21 10:48:05.803934772 +0000 UTC m=+0.018254992 image pull 8f4d013db793795fef09ff5edf96e6b42ba8cd2c9158b2f0b7a5d7c1e657c063 quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472 2024-09-21T10:48:05.937 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:48:05 smithi175 podman[57414]: 2024-09-21 10:48:05.935553727 +0000 UTC m=+0.149873931 container create aab90bc08918acd17479c9d688e560d0ed82ebff8dd25a6dfbd9a2ec00fcafd7 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, ceph=True, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T10:48:06.624 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:48:06 smithi175 podman[57414]: 2024-09-21 10:48:06.191244924 +0000 UTC m=+0.405565135 container init aab90bc08918acd17479c9d688e560d0ed82ebff8dd25a6dfbd9a2ec00fcafd7 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T10:48:06.624 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:48:06 smithi175 podman[57414]: 2024-09-21 10:48:06.195450268 +0000 UTC m=+0.409770463 container start aab90bc08918acd17479c9d688e560d0ed82ebff8dd25a6dfbd9a2ec00fcafd7 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240731, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-09-21T10:48:06.624 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:48:06 smithi175 ceph-osd[57433]: -- 172.21.15.175:0/2598895834 <== mon.1 v2:172.21.15.175:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55cb9677e1e0 con 0x55cb9670e000 2024-09-21T10:48:06.624 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:48:06 smithi175 bash[57414]: aab90bc08918acd17479c9d688e560d0ed82ebff8dd25a6dfbd9a2ec00fcafd7 2024-09-21T10:48:06.624 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:48:06 smithi175 systemd[1]: Started Ceph osd.6 for 3ccd15e6-7804-11ef-baf6-efdc52797490. 2024-09-21T10:48:06.842 INFO:teuthology.orchestra.run.smithi123.stdout: 5 254632 9.58 MB/sec execute 547 sec latency 45.090 ms 2024-09-21T10:48:07.610 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:07 smithi123 ceph-mon[63856]: pgmap v274: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 349 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.4 MiB/s wr, 587 op/s; 196/1665 objects degraded (11.772%) 2024-09-21T10:48:07.610 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:07 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:48:07.610 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:07 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:48:07.611 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:07 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:48:07.611 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:07 smithi123 ceph-mon[63856]: Health check update: Degraded data redundancy: 196/1665 objects degraded (11.772%), 25 pgs degraded (PG_DEGRADED) 2024-09-21T10:48:07.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:07 smithi175 ceph-mon[49304]: pgmap v274: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 349 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.4 MiB/s wr, 587 op/s; 196/1665 objects degraded (11.772%) 2024-09-21T10:48:07.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:07 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:48:07.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:07 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:48:07.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:07 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:48:07.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:07 smithi175 ceph-mon[49304]: Health check update: Degraded data redundancy: 196/1665 objects degraded (11.772%), 25 pgs degraded (PG_DEGRADED) 2024-09-21T10:48:07.842 INFO:teuthology.orchestra.run.smithi123.stdout: 5 255091 9.57 MB/sec execute 548 sec latency 40.703 ms 2024-09-21T10:48:08.373 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:48:08 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6[57429]: 2024-09-21T10:48:08.084+0000 7fc56f19e640 -1 Falling back to public interface 2024-09-21T10:48:08.842 INFO:teuthology.orchestra.run.smithi123.stdout: 5 255562 9.57 MB/sec execute 549 sec latency 33.626 ms 2024-09-21T10:48:09.380 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:09 smithi123 ceph-mon[63856]: pgmap v275: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 349 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.5 MiB/s wr, 443 op/s; 190/1668 objects degraded (11.391%) 2024-09-21T10:48:09.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:09 smithi175 ceph-mon[49304]: pgmap v275: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 349 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.5 MiB/s wr, 443 op/s; 190/1668 objects degraded (11.391%) 2024-09-21T10:48:09.842 INFO:teuthology.orchestra.run.smithi123.stdout: 5 255901 9.57 MB/sec execute 550 sec latency 50.943 ms 2024-09-21T10:48:10.842 INFO:teuthology.orchestra.run.smithi123.stdout: 5 256168 9.58 MB/sec execute 551 sec latency 62.463 ms 2024-09-21T10:48:11.232 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:11 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:48:11.232 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:11 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:48:11.232 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:11 smithi123 ceph-mon[63856]: pgmap v276: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 351 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.5 MiB/s wr, 391 op/s; 186/1695 objects degraded (10.973%) 2024-09-21T10:48:11.232 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:11 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:48:11.232 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:11 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:48:11.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:11 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:48:11.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:11 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:48:11.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:11 smithi175 ceph-mon[49304]: pgmap v276: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 351 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.5 MiB/s wr, 391 op/s; 186/1695 objects degraded (10.973%) 2024-09-21T10:48:11.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:11 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:48:11.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:11 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:48:11.842 INFO:teuthology.orchestra.run.smithi123.stdout: 5 256444 9.57 MB/sec execute 552 sec latency 56.696 ms 2024-09-21T10:48:12.525 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:12 smithi123 ceph-mon[63856]: Health check update: Degraded data redundancy: 183/1683 objects degraded (10.873%), 25 pgs degraded (PG_DEGRADED) 2024-09-21T10:48:12.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:12 smithi175 ceph-mon[49304]: Health check update: Degraded data redundancy: 183/1683 objects degraded (10.873%), 25 pgs degraded (PG_DEGRADED) 2024-09-21T10:48:12.842 INFO:teuthology.orchestra.run.smithi123.stdout: 5 256768 9.57 MB/sec execute 553 sec latency 55.483 ms 2024-09-21T10:48:13.802 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:13 smithi123 ceph-mon[63856]: pgmap v277: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 353 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 4.8 MiB/s wr, 468 op/s; 183/1683 objects degraded (10.873%) 2024-09-21T10:48:13.802 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:13 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:48:13.802 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:13 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:48:13.842 INFO:teuthology.orchestra.run.smithi123.stdout: 5 257136 9.56 MB/sec execute 554 sec latency 45.721 ms 2024-09-21T10:48:13.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:13 smithi175 ceph-mon[49304]: pgmap v277: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 353 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 4.8 MiB/s wr, 468 op/s; 183/1683 objects degraded (10.873%) 2024-09-21T10:48:13.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:13 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:48:13.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:13 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:48:14.842 INFO:teuthology.orchestra.run.smithi123.stdout: 5 257420 9.56 MB/sec execute 555 sec latency 49.930 ms 2024-09-21T10:48:15.629 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:15 smithi123 ceph-mon[63856]: pgmap v278: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 348 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.7 MiB/s wr, 433 op/s; 183/1635 objects degraded (11.193%) 2024-09-21T10:48:15.631 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:15 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:48:15.632 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:15 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:48:15.632 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:15 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:48:15.632 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:15 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:48:15.632 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:15 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:48:15.632 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:15 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:48:15.632 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:15 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:48:15.632 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:15 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:48:15.632 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:15 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:48:15.632 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:15 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:48:15.632 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:15 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-21T10:48:15.632 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:15 smithi123 ceph-mon[63856]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-21T10:48:15.632 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:15 smithi123 ceph-mon[63856]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-21T10:48:15.843 INFO:teuthology.orchestra.run.smithi123.stdout: 5 257775 9.57 MB/sec execute 556 sec latency 46.291 ms 2024-09-21T10:48:15.862 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:15 smithi175 ceph-mon[49304]: pgmap v278: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 348 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.7 MiB/s wr, 433 op/s; 183/1635 objects degraded (11.193%) 2024-09-21T10:48:15.862 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:15 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:48:15.862 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:15 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:48:15.862 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:15 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:48:15.863 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:15 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:48:15.863 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:15 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:48:15.863 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:15 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:48:15.863 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:15 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:48:15.863 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:15 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:48:15.863 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:15 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:48:15.863 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:15 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:48:15.863 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:15 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-21T10:48:15.863 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:15 smithi175 ceph-mon[49304]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-21T10:48:15.863 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:15 smithi175 ceph-mon[49304]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-21T10:48:16.123 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:48:15 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6[57429]: 2024-09-21T10:48:15.858+0000 7fc56f19e640 -1 osd.6 0 read_superblock omap replica is missing. 2024-09-21T10:48:16.843 INFO:teuthology.orchestra.run.smithi123.stdout: 5 258107 9.57 MB/sec execute 557 sec latency 55.087 ms 2024-09-21T10:48:17.387 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:17 smithi123 ceph-mon[63856]: pgmap v279: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 353 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 5.6 MiB/s wr, 581 op/s; 186/1632 objects degraded (11.397%) 2024-09-21T10:48:17.387 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:17 smithi123 ceph-mon[63856]: from='osd.6 [v2:172.21.15.175:6816/1576432591,v1:172.21.15.175:6817/1576432591]' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-21T10:48:17.388 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:17 smithi123 ceph-mon[63856]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-21T10:48:17.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:17 smithi175 ceph-mon[49304]: pgmap v279: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 353 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 5.6 MiB/s wr, 581 op/s; 186/1632 objects degraded (11.397%) 2024-09-21T10:48:17.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:17 smithi175 ceph-mon[49304]: from='osd.6 [v2:172.21.15.175:6816/1576432591,v1:172.21.15.175:6817/1576432591]' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-21T10:48:17.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:17 smithi175 ceph-mon[49304]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-21T10:48:17.624 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:48:17 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6[57429]: 2024-09-21T10:48:17.150+0000 7fc56f19e640 -1 osd.6 153 log_to_monitors true 2024-09-21T10:48:17.843 INFO:teuthology.orchestra.run.smithi123.stdout: 5 258368 9.57 MB/sec execute 558 sec latency 165.415 ms 2024-09-21T10:48:18.123 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:48:17 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6[57429]: 2024-09-21T10:48:17.850+0000 7fc562f40640 -1 osd.6 153 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T10:48:18.448 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:18 smithi123 ceph-mon[63856]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-21T10:48:18.448 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:18 smithi123 ceph-mon[63856]: osdmap e156: 8 total, 7 up, 8 in 2024-09-21T10:48:18.449 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:18 smithi123 ceph-mon[63856]: from='osd.6 [v2:172.21.15.175:6816/1576432591,v1:172.21.15.175:6817/1576432591]' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi175", "root=default"]} : dispatch 2024-09-21T10:48:18.449 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:18 smithi123 ceph-mon[63856]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi175", "root=default"]} : dispatch 2024-09-21T10:48:18.449 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:18 smithi123 ceph-mon[63856]: from='osd.6 ' entity='osd.6' 2024-09-21T10:48:18.843 INFO:teuthology.orchestra.run.smithi123.stdout: 5 258647 9.56 MB/sec execute 559 sec latency 51.794 ms 2024-09-21T10:48:18.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:18 smithi175 ceph-mon[49304]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-21T10:48:18.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:18 smithi175 ceph-mon[49304]: osdmap e156: 8 total, 7 up, 8 in 2024-09-21T10:48:18.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:18 smithi175 ceph-mon[49304]: from='osd.6 [v2:172.21.15.175:6816/1576432591,v1:172.21.15.175:6817/1576432591]' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi175", "root=default"]} : dispatch 2024-09-21T10:48:18.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:18 smithi175 ceph-mon[49304]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi175", "root=default"]} : dispatch 2024-09-21T10:48:18.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:18 smithi175 ceph-mon[49304]: from='osd.6 ' entity='osd.6' 2024-09-21T10:48:19.475 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:19 smithi123 ceph-mon[63856]: pgmap v281: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 350 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.0 MiB/s wr, 502 op/s; 183/1590 objects degraded (11.509%) 2024-09-21T10:48:19.475 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:19 smithi123 ceph-mon[63856]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T10:48:19.475 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:19 smithi123 ceph-mon[63856]: Health check update: Degraded data redundancy: 183/1590 objects degraded (11.509%), 25 pgs degraded (PG_DEGRADED) 2024-09-21T10:48:19.475 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:19 smithi123 ceph-mon[63856]: osd.6 [v2:172.21.15.175:6816/1576432591,v1:172.21.15.175:6817/1576432591] boot 2024-09-21T10:48:19.476 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:19 smithi123 ceph-mon[63856]: osdmap e157: 8 total, 8 up, 8 in 2024-09-21T10:48:19.476 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:19 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-21T10:48:19.843 INFO:teuthology.orchestra.run.smithi123.stdout: 5 258769 9.54 MB/sec execute 560 sec latency 855.668 ms 2024-09-21T10:48:19.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:19 smithi175 ceph-mon[49304]: pgmap v281: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 350 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.0 MiB/s wr, 502 op/s; 183/1590 objects degraded (11.509%) 2024-09-21T10:48:19.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:19 smithi175 ceph-mon[49304]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T10:48:19.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:19 smithi175 ceph-mon[49304]: Health check update: Degraded data redundancy: 183/1590 objects degraded (11.509%), 25 pgs degraded (PG_DEGRADED) 2024-09-21T10:48:19.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:19 smithi175 ceph-mon[49304]: osd.6 [v2:172.21.15.175:6816/1576432591,v1:172.21.15.175:6817/1576432591] boot 2024-09-21T10:48:19.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:19 smithi175 ceph-mon[49304]: osdmap e157: 8 total, 8 up, 8 in 2024-09-21T10:48:19.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:19 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-21T10:48:20.843 INFO:teuthology.orchestra.run.smithi123.stdout: 5 258942 9.53 MB/sec execute 561 sec latency 1697.837 ms 2024-09-21T10:48:20.928 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:20 smithi123 ceph-mon[63856]: osdmap e158: 8 total, 8 up, 8 in 2024-09-21T10:48:20.928 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:20 smithi123 ceph-mon[63856]: pgmap v284: 97 pgs: 5 peering, 9 active+undersized, 21 active+undersized+degraded, 62 active+clean; 353 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.3 MiB/s wr, 610 op/s; 157/1668 objects degraded (9.412%) 2024-09-21T10:48:21.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:20 smithi175 ceph-mon[49304]: osdmap e158: 8 total, 8 up, 8 in 2024-09-21T10:48:21.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:20 smithi175 ceph-mon[49304]: pgmap v284: 97 pgs: 5 peering, 9 active+undersized, 21 active+undersized+degraded, 62 active+clean; 353 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.3 MiB/s wr, 610 op/s; 157/1668 objects degraded (9.412%) 2024-09-21T10:48:21.843 INFO:teuthology.orchestra.run.smithi123.stdout: 5 259170 9.52 MB/sec execute 562 sec latency 1811.170 ms 2024-09-21T10:48:21.958 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:21 smithi123 ceph-mon[63856]: osdmap e159: 8 total, 8 up, 8 in 2024-09-21T10:48:22.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:21 smithi175 ceph-mon[49304]: osdmap e159: 8 total, 8 up, 8 in 2024-09-21T10:48:22.843 INFO:teuthology.orchestra.run.smithi123.stdout: 5 259318 9.51 MB/sec execute 563 sec latency 951.179 ms 2024-09-21T10:48:23.081 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:22 smithi123 ceph-mon[63856]: osdmap e160: 8 total, 8 up, 8 in 2024-09-21T10:48:23.081 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:22 smithi123 ceph-mon[63856]: pgmap v287: 97 pgs: 3 remapped+peering, 5 peering, 2 active+undersized, 14 active+undersized+degraded, 73 active+clean; 353 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.4 MiB/s wr, 420 op/s; 100/1659 objects degraded (6.028%); 45 KiB/s, 11 objects/s recovering 2024-09-21T10:48:23.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:22 smithi175 ceph-mon[49304]: osdmap e160: 8 total, 8 up, 8 in 2024-09-21T10:48:23.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:22 smithi175 ceph-mon[49304]: pgmap v287: 97 pgs: 3 remapped+peering, 5 peering, 2 active+undersized, 14 active+undersized+degraded, 73 active+clean; 353 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.4 MiB/s wr, 420 op/s; 100/1659 objects degraded (6.028%); 45 KiB/s, 11 objects/s recovering 2024-09-21T10:48:23.843 INFO:teuthology.orchestra.run.smithi123.stdout: 5 259610 9.52 MB/sec execute 564 sec latency 999.860 ms 2024-09-21T10:48:24.098 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:23 smithi123 ceph-mon[63856]: osdmap e161: 8 total, 8 up, 8 in 2024-09-21T10:48:24.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:23 smithi175 ceph-mon[49304]: osdmap e161: 8 total, 8 up, 8 in 2024-09-21T10:48:24.843 INFO:teuthology.orchestra.run.smithi123.stdout: 5 259953 9.52 MB/sec execute 565 sec latency 59.449 ms 2024-09-21T10:48:25.115 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:24 smithi123 ceph-mon[63856]: pgmap v289: 97 pgs: 2 remapped+peering, 7 peering, 88 active+clean; 346 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 2.8 MiB/s wr, 172 op/s; 141 KiB/s, 52 keys/s, 34 objects/s recovering 2024-09-21T10:48:25.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:24 smithi175 ceph-mon[49304]: pgmap v289: 97 pgs: 2 remapped+peering, 7 peering, 88 active+clean; 346 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 2.8 MiB/s wr, 172 op/s; 141 KiB/s, 52 keys/s, 34 objects/s recovering 2024-09-21T10:48:25.843 INFO:teuthology.orchestra.run.smithi123.stdout: 5 260387 9.52 MB/sec execute 566 sec latency 44.914 ms 2024-09-21T10:48:26.128 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:25 smithi123 ceph-mon[63856]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 100/1659 objects degraded (6.028%), 14 pgs degraded) 2024-09-21T10:48:26.128 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:25 smithi123 ceph-mon[63856]: Cluster is now healthy 2024-09-21T10:48:26.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:25 smithi175 ceph-mon[49304]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 100/1659 objects degraded (6.028%), 14 pgs degraded) 2024-09-21T10:48:26.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:25 smithi175 ceph-mon[49304]: Cluster is now healthy 2024-09-21T10:48:26.843 INFO:teuthology.orchestra.run.smithi123.stdout: 5 260650 9.51 MB/sec execute 567 sec latency 90.187 ms 2024-09-21T10:48:27.143 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:26 smithi123 ceph-mon[63856]: pgmap v290: 97 pgs: 2 peering, 95 active+clean; 350 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 3.8 MiB/s wr, 414 op/s; 968 KiB/s, 39 keys/s, 41 objects/s recovering 2024-09-21T10:48:27.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:26 smithi175 ceph-mon[49304]: pgmap v290: 97 pgs: 2 peering, 95 active+clean; 350 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 3.8 MiB/s wr, 414 op/s; 968 KiB/s, 39 keys/s, 41 objects/s recovering 2024-09-21T10:48:27.843 INFO:teuthology.orchestra.run.smithi123.stdout: 5 260941 9.51 MB/sec execute 568 sec latency 72.681 ms 2024-09-21T10:48:28.675 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:28 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:48:28.675 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:28 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:48:28.843 INFO:teuthology.orchestra.run.smithi123.stdout: 5 261274 9.51 MB/sec execute 569 sec latency 72.315 ms 2024-09-21T10:48:29.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:28 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:48:29.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:28 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:48:29.722 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:29 smithi123 ceph-mon[63856]: pgmap v291: 97 pgs: 97 active+clean; 345 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 2.9 MiB/s wr, 340 op/s; 1.4 MiB/s, 31 keys/s, 30 objects/s recovering 2024-09-21T10:48:29.843 INFO:teuthology.orchestra.run.smithi123.stdout: 5 261606 9.52 MB/sec execute 570 sec latency 69.844 ms 2024-09-21T10:48:30.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:29 smithi175 ceph-mon[49304]: pgmap v291: 97 pgs: 97 active+clean; 345 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 2.9 MiB/s wr, 340 op/s; 1.4 MiB/s, 31 keys/s, 30 objects/s recovering 2024-09-21T10:48:30.738 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:30 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-21T10:48:30.738 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:30 smithi123 ceph-mon[63856]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-21T10:48:30.738 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:30 smithi123 ceph-mon[63856]: Upgrade: osd.7 is safe to restart 2024-09-21T10:48:30.843 INFO:teuthology.orchestra.run.smithi123.stdout: 5 261896 9.51 MB/sec execute 571 sec latency 79.265 ms 2024-09-21T10:48:30.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:30 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-21T10:48:30.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:30 smithi175 ceph-mon[49304]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-21T10:48:30.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:30 smithi175 ceph-mon[49304]: Upgrade: osd.7 is safe to restart 2024-09-21T10:48:31.773 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:31 smithi123 ceph-mon[63856]: pgmap v292: 97 pgs: 97 active+clean; 346 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 9.1 KiB/s rd, 3.8 MiB/s wr, 351 op/s; 1.3 MiB/s, 27 keys/s, 27 objects/s recovering 2024-09-21T10:48:31.773 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:31 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:48:31.773 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:31 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-21T10:48:31.773 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:31 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:48:31.843 INFO:teuthology.orchestra.run.smithi123.stdout: 5 262146 9.51 MB/sec execute 572 sec latency 40.165 ms 2024-09-21T10:48:31.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:31 smithi175 ceph-mon[49304]: pgmap v292: 97 pgs: 97 active+clean; 346 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 9.1 KiB/s rd, 3.8 MiB/s wr, 351 op/s; 1.3 MiB/s, 27 keys/s, 27 objects/s recovering 2024-09-21T10:48:31.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:31 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:48:31.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:31 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-21T10:48:31.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:31 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:48:32.802 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:32 smithi123 ceph-mon[63856]: Upgrade: Updating osd.7 2024-09-21T10:48:32.802 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:32 smithi123 ceph-mon[63856]: Deploying daemon osd.7 on smithi175 2024-09-21T10:48:32.843 INFO:teuthology.orchestra.run.smithi123.stdout: 5 262370 9.49 MB/sec execute 573 sec latency 57.211 ms 2024-09-21T10:48:32.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:32 smithi175 ceph-mon[49304]: Upgrade: Updating osd.7 2024-09-21T10:48:32.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:32 smithi175 ceph-mon[49304]: Deploying daemon osd.7 on smithi175 2024-09-21T10:48:33.635 INFO:teuthology.orchestra.run.smithi123.stdout:true 2024-09-21T10:48:33.824 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:33 smithi123 ceph-mon[63856]: pgmap v293: 97 pgs: 97 active+clean; 351 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.3 MiB/s wr, 400 op/s; 1.1 MiB/s, 23 keys/s, 22 objects/s recovering 2024-09-21T10:48:33.843 INFO:teuthology.orchestra.run.smithi123.stdout: 5 262598 9.49 MB/sec execute 574 sec latency 64.525 ms 2024-09-21T10:48:33.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:33 smithi175 ceph-mon[49304]: pgmap v293: 97 pgs: 97 active+clean; 351 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.3 MiB/s wr, 400 op/s; 1.1 MiB/s, 23 keys/s, 22 objects/s recovering 2024-09-21T10:48:34.501 INFO:teuthology.orchestra.run.smithi123.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T10:48:34.501 INFO:teuthology.orchestra.run.smithi123.stdout:alertmanager.smithi123 smithi123 *:9093,9094 running (9m) 2m ago 18m 22.0M - 0.27.0 11f11916f8cd 5d550bd97de5 2024-09-21T10:48:34.501 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi123 smithi123 running (5m) 2m ago 18m 6899k - 19.3.0-5103-g5c1cdc3c 8f4d013db793 c5e1ef39a0a8 2024-09-21T10:48:34.502 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi175 smithi175 running (5m) 24s ago 17m 6895k - 19.3.0-5103-g5c1cdc3c 8f4d013db793 fb1ca999ccfd 2024-09-21T10:48:34.502 INFO:teuthology.orchestra.run.smithi123.stdout:grafana.smithi123 smithi123 *:3000 running (8m) 2m ago 17m 52.6M - dad864ee21e9 771df81763ba 2024-09-21T10:48:34.502 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi123.uqoyow smithi123 running (12m) 2m ago 12m 41.2M - 17.2.0 e1d6a67b021e f0a21eaac266 2024-09-21T10:48:34.502 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi175.jrlevw smithi175 running (12m) 24s ago 12m 14.5M - 17.2.0 e1d6a67b021e b21b16f4bc50 2024-09-21T10:48:34.502 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi123.stnuit smithi123 *:8443,9283,8765 running (8m) 2m ago 19m 519M - 19.3.0-5103-g5c1cdc3c 8f4d013db793 6812cc5cb56b 2024-09-21T10:48:34.502 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi175.txvwrq smithi175 *:8443,9283,8765 running (7m) 24s ago 17m 456M - 19.3.0-5103-g5c1cdc3c 8f4d013db793 304f0408075e 2024-09-21T10:48:34.502 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi123 smithi123 running (7m) 2m ago 19m 52.0M 2048M 19.3.0-5103-g5c1cdc3c 8f4d013db793 6b95207a2d80 2024-09-21T10:48:34.502 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi175 smithi175 running (6m) 24s ago 17m 46.0M 2048M 19.3.0-5103-g5c1cdc3c 8f4d013db793 3abee2d37f0f 2024-09-21T10:48:34.502 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.0.0.smithi123.ddatpr smithi123 *:2049 running (11m) 2m ago 11m 281M - 4.0 e1d6a67b021e 36f8cb2281eb 2024-09-21T10:48:34.502 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.1.0.smithi175.wpawer smithi175 *:2049 running (11m) 24s ago 11m 90.9M - 4.0 e1d6a67b021e 2200aeed9200 2024-09-21T10:48:34.502 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi123 smithi123 *:9100 running (8m) 2m ago 17m 12.3M - 1.7.0 72c9c2088986 ca7d0b29af81 2024-09-21T10:48:34.502 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi175 smithi175 *:9100 running (8m) 24s ago 17m 11.8M - 1.7.0 72c9c2088986 46c4f6b7c149 2024-09-21T10:48:34.502 INFO:teuthology.orchestra.run.smithi123.stdout:osd.0 smithi123 running (5m) 2m ago 16m 362M 1936M 19.3.0-5103-g5c1cdc3c 8f4d013db793 cc40f695c3a8 2024-09-21T10:48:34.503 INFO:teuthology.orchestra.run.smithi123.stdout:osd.1 smithi123 running (4m) 2m ago 16m 279M 1936M 19.3.0-5103-g5c1cdc3c 8f4d013db793 1be249025c7f 2024-09-21T10:48:34.503 INFO:teuthology.orchestra.run.smithi123.stdout:osd.2 smithi123 running (3m) 2m ago 15m 232M 1936M 19.3.0-5103-g5c1cdc3c 8f4d013db793 647aec759e3c 2024-09-21T10:48:34.503 INFO:teuthology.orchestra.run.smithi123.stdout:osd.3 smithi123 running (2m) 2m ago 15m 11.6M 1936M 19.3.0-5103-g5c1cdc3c 8f4d013db793 7a082fcad8e8 2024-09-21T10:48:34.503 INFO:teuthology.orchestra.run.smithi123.stdout:osd.4 smithi175 running (105s) 24s ago 14m 306M 2704M 19.3.0-5103-g5c1cdc3c 8f4d013db793 b650f72d10f0 2024-09-21T10:48:34.503 INFO:teuthology.orchestra.run.smithi123.stdout:osd.5 smithi175 running (68s) 24s ago 14m 257M 2704M 19.3.0-5103-g5c1cdc3c 8f4d013db793 8199a0b362df 2024-09-21T10:48:34.503 INFO:teuthology.orchestra.run.smithi123.stdout:osd.6 smithi175 running (28s) 24s ago 14m 12.1M 2704M 19.3.0-5103-g5c1cdc3c 8f4d013db793 aab90bc08918 2024-09-21T10:48:34.503 INFO:teuthology.orchestra.run.smithi123.stdout:osd.7 smithi175 running (13m) 24s ago 13m 798M 2704M 17.2.0 e1d6a67b021e e9b8f1f90994 2024-09-21T10:48:34.503 INFO:teuthology.orchestra.run.smithi123.stdout:prometheus.smithi123 smithi123 *:9095 running (7m) 2m ago 17m 55.4M - 2.51.0 1d3b7f56885b 37f944c9de88 2024-09-21T10:48:34.672 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:48:34 smithi175 systemd[1]: Stopping Ceph osd.7 for 3ccd15e6-7804-11ef-baf6-efdc52797490... 2024-09-21T10:48:34.843 INFO:teuthology.orchestra.run.smithi123.stdout: 5 262808 9.48 MB/sec execute 575 sec latency 66.539 ms 2024-09-21T10:48:34.882 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:34 smithi123 ceph-mon[63856]: from='client.34434 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:48:35.020 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-09-21T10:48:35.020 INFO:teuthology.orchestra.run.smithi123.stdout: "mon": { 2024-09-21T10:48:35.020 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev)": 2 2024-09-21T10:48:35.020 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:48:35.020 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr": { 2024-09-21T10:48:35.020 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev)": 2 2024-09-21T10:48:35.020 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:48:35.020 INFO:teuthology.orchestra.run.smithi123.stdout: "osd": { 2024-09-21T10:48:35.021 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-21T10:48:35.021 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev)": 7 2024-09-21T10:48:35.021 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:48:35.021 INFO:teuthology.orchestra.run.smithi123.stdout: "mds": { 2024-09-21T10:48:35.021 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T10:48:35.021 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:48:35.021 INFO:teuthology.orchestra.run.smithi123.stdout: "overall": { 2024-09-21T10:48:35.021 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-09-21T10:48:35.021 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev)": 11 2024-09-21T10:48:35.021 INFO:teuthology.orchestra.run.smithi123.stdout: } 2024-09-21T10:48:35.021 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-09-21T10:48:35.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:34 smithi175 ceph-mon[49304]: from='client.34434 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:48:35.123 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:48:34 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[38891]: 2024-09-21T10:48:34.841+0000 7fbd23d4c700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:48:35.124 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:48:34 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[38891]: 2024-09-21T10:48:34.841+0000 7fbd23d4c700 -1 osd.7 161 *** Got signal Terminated *** 2024-09-21T10:48:35.124 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:48:34 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[38891]: 2024-09-21T10:48:34.841+0000 7fbd23d4c700 -1 osd.7 161 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T10:48:35.481 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-09-21T10:48:35.482 INFO:teuthology.orchestra.run.smithi123.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472", 2024-09-21T10:48:35.482 INFO:teuthology.orchestra.run.smithi123.stdout: "in_progress": true, 2024-09-21T10:48:35.482 INFO:teuthology.orchestra.run.smithi123.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T10:48:35.482 INFO:teuthology.orchestra.run.smithi123.stdout: "services_complete": [ 2024-09-21T10:48:35.482 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr", 2024-09-21T10:48:35.482 INFO:teuthology.orchestra.run.smithi123.stdout: "crash", 2024-09-21T10:48:35.483 INFO:teuthology.orchestra.run.smithi123.stdout: "mon" 2024-09-21T10:48:35.483 INFO:teuthology.orchestra.run.smithi123.stdout: ], 2024-09-21T10:48:35.483 INFO:teuthology.orchestra.run.smithi123.stdout: "progress": "13/23 daemons upgraded", 2024-09-21T10:48:35.483 INFO:teuthology.orchestra.run.smithi123.stdout: "message": "Currently upgrading osd daemons", 2024-09-21T10:48:35.483 INFO:teuthology.orchestra.run.smithi123.stdout: "is_paused": false 2024-09-21T10:48:35.483 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-09-21T10:48:35.843 INFO:teuthology.orchestra.run.smithi123.stdout: 5 263000 9.48 MB/sec execute 576 sec latency 200.628 ms 2024-09-21T10:48:35.960 INFO:teuthology.orchestra.run.smithi123.stdout:HEALTH_WARN 1 osds down; all OSDs are running squid or later but require_osd_release < squid 2024-09-21T10:48:35.961 INFO:teuthology.orchestra.run.smithi123.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-21T10:48:35.961 INFO:teuthology.orchestra.run.smithi123.stdout: osd.7 (root=default,host=smithi175) is down 2024-09-21T10:48:35.961 INFO:teuthology.orchestra.run.smithi123.stdout:[WRN] OSD_UPGRADE_FINISHED: all OSDs are running squid or later but require_osd_release < squid 2024-09-21T10:48:35.961 INFO:teuthology.orchestra.run.smithi123.stdout: all OSDs are running squid or later but require_osd_release < squid 2024-09-21T10:48:35.961 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:35 smithi123 ceph-mon[63856]: from='client.34438 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:48:35.961 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:35 smithi123 ceph-mon[63856]: pgmap v294: 97 pgs: 97 active+clean; 349 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.7 MiB/s wr, 492 op/s; 939 KiB/s, 20 keys/s, 17 objects/s recovering 2024-09-21T10:48:35.961 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:35 smithi123 ceph-mon[63856]: from='client.34442 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:48:35.961 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:35 smithi123 ceph-mon[63856]: osd.7 marked itself down and dead 2024-09-21T10:48:35.961 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:35 smithi123 ceph-mon[63856]: from='client.? 172.21.15.123:0/3317973765' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:48:36.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:35 smithi175 ceph-mon[49304]: from='client.34438 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:48:36.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:35 smithi175 ceph-mon[49304]: pgmap v294: 97 pgs: 97 active+clean; 349 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.7 MiB/s wr, 492 op/s; 939 KiB/s, 20 keys/s, 17 objects/s recovering 2024-09-21T10:48:36.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:35 smithi175 ceph-mon[49304]: from='client.34442 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:48:36.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:35 smithi175 ceph-mon[49304]: osd.7 marked itself down and dead 2024-09-21T10:48:36.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:35 smithi175 ceph-mon[49304]: from='client.? 172.21.15.123:0/3317973765' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:48:36.623 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:48:36 smithi175 podman[59003]: 2024-09-21 10:48:36.357987525 +0000 UTC m=+1.790543746 container died e9b8f1f90994b9444855214c027e902528af2168bdccb7df831456de3ea83b26 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7, com.redhat.component=centos-stream-container, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, GIT_CLEAN=True, name=centos-stream, maintainer=Guillaume Abrioux , distribution-scope=public, io.openshift.expose-services=, vcs-type=git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, version=8, ceph=True, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc.) 2024-09-21T10:48:36.843 INFO:teuthology.orchestra.run.smithi123.stdout: 5 263051 9.47 MB/sec execute 577 sec latency 1088.442 ms 2024-09-21T10:48:36.951 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:36 smithi123 ceph-mon[63856]: from='client.34450 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:48:36.951 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:36 smithi123 ceph-mon[63856]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T10:48:36.951 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:36 smithi123 ceph-mon[63856]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-09-21T10:48:36.951 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:36 smithi123 ceph-mon[63856]: osdmap e162: 8 total, 7 up, 8 in 2024-09-21T10:48:36.951 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:36 smithi123 ceph-mon[63856]: from='client.? 172.21.15.123:0/2520003353' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T10:48:36.952 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:36 smithi123 ceph-mon[63856]: osdmap e163: 8 total, 7 up, 8 in 2024-09-21T10:48:36.961 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:48:36 smithi175 podman[59003]: 2024-09-21 10:48:36.749499626 +0000 UTC m=+2.182055834 container cleanup e9b8f1f90994b9444855214c027e902528af2168bdccb7df831456de3ea83b26 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7, ceph=True, io.buildah.version=1.19.8, release=754, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, name=centos-stream, distribution-scope=public, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-09-21T10:48:36.961 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:48:36 smithi175 bash[59003]: ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7 2024-09-21T10:48:36.961 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:36 smithi175 ceph-mon[49304]: from='client.34450 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:48:36.961 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:36 smithi175 ceph-mon[49304]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T10:48:36.961 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:36 smithi175 ceph-mon[49304]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-09-21T10:48:36.961 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:36 smithi175 ceph-mon[49304]: osdmap e162: 8 total, 7 up, 8 in 2024-09-21T10:48:36.961 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:36 smithi175 ceph-mon[49304]: from='client.? 172.21.15.123:0/2520003353' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T10:48:36.961 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:36 smithi175 ceph-mon[49304]: osdmap e163: 8 total, 7 up, 8 in 2024-09-21T10:48:37.368 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:48:36 smithi175 podman[59017]: 2024-09-21 10:48:36.959385323 +0000 UTC m=+0.602399269 container remove e9b8f1f90994b9444855214c027e902528af2168bdccb7df831456de3ea83b26 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, distribution-scope=public, io.buildah.version=1.19.8, version=8, vcs-type=git, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870) 2024-09-21T10:48:37.623 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:48:37 smithi175 podman[59110]: 2024-09-21 10:48:37.269778968 +0000 UTC m=+0.017281912 image pull 8f4d013db793795fef09ff5edf96e6b42ba8cd2c9158b2f0b7a5d7c1e657c063 quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472 2024-09-21T10:48:37.623 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:48:37 smithi175 podman[59110]: 2024-09-21 10:48:37.394615656 +0000 UTC m=+0.142118635 container create 864a2abb1d6f187346bf2118629bb82701fd3b70203cf33c8451743afd39e23e (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7-deactivate, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-09-21T10:48:37.714 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:37 smithi123 ceph-mon[63856]: pgmap v296: 97 pgs: 8 peering, 7 stale+active+clean, 82 active+clean; 343 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.3 MiB/s wr, 375 op/s; 526 KiB/s, 2 objects/s recovering 2024-09-21T10:48:37.843 INFO:teuthology.orchestra.run.smithi123.stdout: 5 263403 9.47 MB/sec execute 578 sec latency 62.347 ms 2024-09-21T10:48:38.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:37 smithi175 ceph-mon[49304]: pgmap v296: 97 pgs: 8 peering, 7 stale+active+clean, 82 active+clean; 343 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.3 MiB/s wr, 375 op/s; 526 KiB/s, 2 objects/s recovering 2024-09-21T10:48:38.124 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:48:37 smithi175 podman[59110]: 2024-09-21 10:48:37.672895156 +0000 UTC m=+0.420398092 container init 864a2abb1d6f187346bf2118629bb82701fd3b70203cf33c8451743afd39e23e (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7-deactivate, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True) 2024-09-21T10:48:38.124 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:48:37 smithi175 podman[59110]: 2024-09-21 10:48:37.676353595 +0000 UTC m=+0.423856528 container start 864a2abb1d6f187346bf2118629bb82701fd3b70203cf33c8451743afd39e23e (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7-deactivate, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, ceph=True) 2024-09-21T10:48:38.124 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:48:37 smithi175 podman[59110]: 2024-09-21 10:48:37.781582415 +0000 UTC m=+0.529085350 container attach 864a2abb1d6f187346bf2118629bb82701fd3b70203cf33c8451743afd39e23e (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.37.2) 2024-09-21T10:48:38.124 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:48:37 smithi175 podman[59110]: 2024-09-21 10:48:37.875848819 +0000 UTC m=+0.623351760 container died 864a2abb1d6f187346bf2118629bb82701fd3b70203cf33c8451743afd39e23e (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7-deactivate, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240731) 2024-09-21T10:48:38.843 INFO:teuthology.orchestra.run.smithi123.stdout: 5 263788 9.47 MB/sec execute 579 sec latency 44.448 ms 2024-09-21T10:48:38.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:38 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:48:38.873 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:48:38 smithi175 podman[59134]: 2024-09-21 10:48:38.544700159 +0000 UTC m=+0.662779555 container remove 864a2abb1d6f187346bf2118629bb82701fd3b70203cf33c8451743afd39e23e (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7-deactivate, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3) 2024-09-21T10:48:38.874 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:48:38 smithi175 systemd[1]: ceph-3ccd15e6-7804-11ef-baf6-efdc52797490@osd.7.service: Deactivated successfully. 2024-09-21T10:48:38.874 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:48:38 smithi175 systemd[1]: Stopped Ceph osd.7 for 3ccd15e6-7804-11ef-baf6-efdc52797490. 2024-09-21T10:48:38.874 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:48:38 smithi175 systemd[1]: ceph-3ccd15e6-7804-11ef-baf6-efdc52797490@osd.7.service: Consumed 6min 33.599s CPU time. 2024-09-21T10:48:38.874 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:48:38 smithi175 systemd[1]: Starting Ceph osd.7 for 3ccd15e6-7804-11ef-baf6-efdc52797490... 2024-09-21T10:48:38.969 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:38 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:48:39.276 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:48:38 smithi175 podman[59233]: 2024-09-21 10:48:38.888908366 +0000 UTC m=+0.016418077 image pull 8f4d013db793795fef09ff5edf96e6b42ba8cd2c9158b2f0b7a5d7c1e657c063 quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472 2024-09-21T10:48:39.276 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:48:39 smithi175 podman[59233]: 2024-09-21 10:48:39.013011543 +0000 UTC m=+0.140521251 container create 3635ed3ff13bbb9d1cd08846e4324547070a74aac537c0cc1f001f48be09dc1b (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7-activate, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-09-21T10:48:39.276 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:48:39 smithi175 podman[59233]: 2024-09-21 10:48:39.274491133 +0000 UTC m=+0.402000833 container init 3635ed3ff13bbb9d1cd08846e4324547070a74aac537c0cc1f001f48be09dc1b (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7-activate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_CLEAN=True) 2024-09-21T10:48:39.623 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:48:39 smithi175 podman[59233]: 2024-09-21 10:48:39.277960547 +0000 UTC m=+0.405470251 container start 3635ed3ff13bbb9d1cd08846e4324547070a74aac537c0cc1f001f48be09dc1b (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731) 2024-09-21T10:48:39.624 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:48:39 smithi175 podman[59233]: 2024-09-21 10:48:39.374679572 +0000 UTC m=+0.502189298 container attach 3635ed3ff13bbb9d1cd08846e4324547070a74aac537c0cc1f001f48be09dc1b (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7-activate, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T10:48:39.624 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:48:39 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7-activate[59247]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:48:39.624 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:48:39 smithi175 bash[59233]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:48:39.624 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:48:39 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7-activate[59247]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:48:39.624 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:48:39 smithi175 bash[59233]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:48:39.843 INFO:teuthology.orchestra.run.smithi123.stdout: 5 264106 9.46 MB/sec execute 580 sec latency 53.289 ms 2024-09-21T10:48:39.978 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:39 smithi123 ceph-mon[63856]: pgmap v298: 97 pgs: 8 peering, 6 stale+active+clean, 8 active+undersized+degraded, 75 active+clean; 342 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 4.5 MiB/s wr, 531 op/s; 28/1554 objects degraded (1.802%); 4.8 KiB/s, 0 objects/s recovering 2024-09-21T10:48:39.978 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:39 smithi123 ceph-mon[63856]: Health check failed: Degraded data redundancy: 28/1554 objects degraded (1.802%), 8 pgs degraded (PG_DEGRADED) 2024-09-21T10:48:40.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:39 smithi175 ceph-mon[49304]: pgmap v298: 97 pgs: 8 peering, 6 stale+active+clean, 8 active+undersized+degraded, 75 active+clean; 342 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 4.5 MiB/s wr, 531 op/s; 28/1554 objects degraded (1.802%); 4.8 KiB/s, 0 objects/s recovering 2024-09-21T10:48:40.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:39 smithi175 ceph-mon[49304]: Health check failed: Degraded data redundancy: 28/1554 objects degraded (1.802%), 8 pgs degraded (PG_DEGRADED) 2024-09-21T10:48:40.843 INFO:teuthology.orchestra.run.smithi123.stdout: 5 264364 9.45 MB/sec execute 581 sec latency 49.438 ms 2024-09-21T10:48:41.796 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:41 smithi123 ceph-mon[63856]: pgmap v299: 97 pgs: 5 active+undersized, 8 peering, 1 stale+active+clean, 21 active+undersized+degraded, 62 active+clean; 341 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 3.5 MiB/s wr, 428 op/s; 131/1575 objects degraded (8.317%) 2024-09-21T10:48:41.843 INFO:teuthology.orchestra.run.smithi123.stdout: 5 264617 9.45 MB/sec execute 582 sec latency 86.612 ms 2024-09-21T10:48:42.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:41 smithi175 ceph-mon[49304]: pgmap v299: 97 pgs: 5 active+undersized, 8 peering, 1 stale+active+clean, 21 active+undersized+degraded, 62 active+clean; 341 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 3.5 MiB/s wr, 428 op/s; 131/1575 objects degraded (8.317%) 2024-09-21T10:48:42.623 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:48:42 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7-activate[59247]: --> Failed to activate via raw: did not find any matching OSD to activate 2024-09-21T10:48:42.624 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:48:42 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7-activate[59247]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:48:42.624 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:48:42 smithi175 bash[59233]: --> Failed to activate via raw: did not find any matching OSD to activate 2024-09-21T10:48:42.624 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:48:42 smithi175 bash[59233]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:48:42.624 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:48:42 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7-activate[59247]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:48:42.624 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:48:42 smithi175 bash[59233]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T10:48:42.624 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:48:42 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7-activate[59247]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-21T10:48:42.624 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:48:42 smithi175 bash[59233]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-21T10:48:42.624 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:48:42 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7-activate[59247]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_1 --path /var/lib/ceph/osd/ceph-7 --no-mon-config 2024-09-21T10:48:42.624 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:48:42 smithi175 bash[59233]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_1 --path /var/lib/ceph/osd/ceph-7 --no-mon-config 2024-09-21T10:48:42.843 INFO:teuthology.orchestra.run.smithi123.stdout: 5 264989 9.46 MB/sec execute 583 sec latency 41.664 ms 2024-09-21T10:48:43.071 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:48:42 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7-activate[59247]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-7/block 2024-09-21T10:48:43.071 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:48:42 smithi175 bash[59233]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-7/block 2024-09-21T10:48:43.071 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:48:42 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7-activate[59247]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-7/block 2024-09-21T10:48:43.072 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:48:42 smithi175 bash[59233]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-7/block 2024-09-21T10:48:43.072 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:48:42 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7-activate[59247]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-21T10:48:43.072 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:48:42 smithi175 bash[59233]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-21T10:48:43.072 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:48:42 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7-activate[59247]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-21T10:48:43.072 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:48:42 smithi175 bash[59233]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-21T10:48:43.072 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:48:42 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7-activate[59247]: --> ceph-volume lvm activate successful for osd ID: 7 2024-09-21T10:48:43.072 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:48:42 smithi175 bash[59233]: --> ceph-volume lvm activate successful for osd ID: 7 2024-09-21T10:48:43.072 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:48:42 smithi175 podman[59233]: 2024-09-21 10:48:42.75932733 +0000 UTC m=+3.886837031 container died 3635ed3ff13bbb9d1cd08846e4324547070a74aac537c0cc1f001f48be09dc1b (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7-activate, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3) 2024-09-21T10:48:43.649 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:43 smithi175 ceph-mon[49304]: pgmap v300: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 343 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.3 MiB/s wr, 260 op/s; 266/1635 objects degraded (16.269%) 2024-09-21T10:48:43.649 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:43 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:48:43.649 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:43 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:48:43.650 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:48:43 smithi175 podman[59451]: 2024-09-21 10:48:43.379462554 +0000 UTC m=+0.610752480 container remove 3635ed3ff13bbb9d1cd08846e4324547070a74aac537c0cc1f001f48be09dc1b (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7-activate, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, GIT_CLEAN=True) 2024-09-21T10:48:43.650 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:48:43 smithi175 podman[59507]: 2024-09-21 10:48:43.504715887 +0000 UTC m=+0.019296216 image pull 8f4d013db793795fef09ff5edf96e6b42ba8cd2c9158b2f0b7a5d7c1e657c063 quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472 2024-09-21T10:48:43.843 INFO:teuthology.orchestra.run.smithi123.stdout: 5 265435 9.47 MB/sec execute 584 sec latency 49.479 ms 2024-09-21T10:48:43.874 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:43 smithi123 ceph-mon[63856]: pgmap v300: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 343 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.3 MiB/s wr, 260 op/s; 266/1635 objects degraded (16.269%) 2024-09-21T10:48:43.874 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:43 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:48:43.874 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:43 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:48:43.901 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:48:43 smithi175 podman[59507]: 2024-09-21 10:48:43.647590856 +0000 UTC m=+0.162171193 container create 2f37c8ff343fc4f62a1d63a7013f5f1b77ab94b59eb6256adb6bc1af62e8d2d2 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.schema-version=1.0) 2024-09-21T10:48:44.373 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:48:43 smithi175 podman[59507]: 2024-09-21 10:48:43.900363014 +0000 UTC m=+0.414943369 container init 2f37c8ff343fc4f62a1d63a7013f5f1b77ab94b59eb6256adb6bc1af62e8d2d2 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=) 2024-09-21T10:48:44.373 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:48:43 smithi175 podman[59507]: 2024-09-21 10:48:43.903974943 +0000 UTC m=+0.418555272 container start 2f37c8ff343fc4f62a1d63a7013f5f1b77ab94b59eb6256adb6bc1af62e8d2d2 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731) 2024-09-21T10:48:44.374 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:48:43 smithi175 ceph-osd[59526]: -- 172.21.15.175:0/4184602285 <== mon.1 v2:172.21.15.175:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5582a4c061e0 con 0x5582a4b92000 2024-09-21T10:48:44.374 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:48:44 smithi175 bash[59507]: 2f37c8ff343fc4f62a1d63a7013f5f1b77ab94b59eb6256adb6bc1af62e8d2d2 2024-09-21T10:48:44.374 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:48:44 smithi175 systemd[1]: Started Ceph osd.7 for 3ccd15e6-7804-11ef-baf6-efdc52797490. 2024-09-21T10:48:44.844 INFO:teuthology.orchestra.run.smithi123.stdout: 5 265732 9.46 MB/sec execute 585 sec latency 54.908 ms 2024-09-21T10:48:45.141 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:45 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:48:45.141 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:45 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:48:45.141 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:45 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:48:45.141 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:45 smithi123 ceph-mon[63856]: pgmap v301: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 343 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 3.1 MiB/s wr, 319 op/s; 266/1656 objects degraded (16.063%) 2024-09-21T10:48:45.141 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:45 smithi123 ceph-mon[63856]: Health check update: Degraded data redundancy: 266/1635 objects degraded (16.269%), 35 pgs degraded (PG_DEGRADED) 2024-09-21T10:48:45.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:45 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:48:45.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:45 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:48:45.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:45 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:48:45.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:45 smithi175 ceph-mon[49304]: pgmap v301: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 343 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 3.1 MiB/s wr, 319 op/s; 266/1656 objects degraded (16.063%) 2024-09-21T10:48:45.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:45 smithi175 ceph-mon[49304]: Health check update: Degraded data redundancy: 266/1635 objects degraded (16.269%), 35 pgs degraded (PG_DEGRADED) 2024-09-21T10:48:45.844 INFO:teuthology.orchestra.run.smithi123.stdout: 5 266114 9.46 MB/sec execute 586 sec latency 53.467 ms 2024-09-21T10:48:45.844 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:48:45 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:48:45.780+0000 7f3062f27640 -1 Falling back to public interface 2024-09-21T10:48:46.844 INFO:teuthology.orchestra.run.smithi123.stdout: 5 266423 9.45 MB/sec execute 587 sec latency 40.781 ms 2024-09-21T10:48:47.429 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:47 smithi123 ceph-mon[63856]: pgmap v302: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 346 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 4.2 MiB/s wr, 435 op/s; 265/1683 objects degraded (15.746%) 2024-09-21T10:48:47.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:47 smithi175 ceph-mon[49304]: pgmap v302: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 346 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 4.2 MiB/s wr, 435 op/s; 265/1683 objects degraded (15.746%) 2024-09-21T10:48:47.844 INFO:teuthology.orchestra.run.smithi123.stdout: 5 266847 9.46 MB/sec execute 588 sec latency 38.373 ms 2024-09-21T10:48:48.844 INFO:teuthology.orchestra.run.smithi123.stdout: 5 267336 9.47 MB/sec execute 589 sec latency 43.410 ms 2024-09-21T10:48:48.946 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:48 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:48:48.946 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:48 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:48:48.947 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:48 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:48:48.947 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:48 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:48:49.043 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:48 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:48:49.043 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:48 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:48:49.043 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:48 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:48:49.043 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:48 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:48:49.844 INFO:teuthology.orchestra.run.smithi123.stdout: 5 267766 9.46 MB/sec execute 590 sec latency 36.467 ms 2024-09-21T10:48:49.963 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:49 smithi123 ceph-mon[63856]: pgmap v303: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 347 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 3.9 MiB/s wr, 389 op/s; 265/1707 objects degraded (15.524%) 2024-09-21T10:48:50.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:49 smithi175 ceph-mon[49304]: pgmap v303: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 347 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 3.9 MiB/s wr, 389 op/s; 265/1707 objects degraded (15.524%) 2024-09-21T10:48:50.842 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:50 smithi123 ceph-mon[63856]: Health check update: Degraded data redundancy: 265/1707 objects degraded (15.524%), 35 pgs degraded (PG_DEGRADED) 2024-09-21T10:48:50.843 INFO:teuthology.orchestra.run.smithi123.stdout: 5 268126 9.47 MB/sec execute 591 sec latency 43.436 ms 2024-09-21T10:48:50.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:50 smithi175 ceph-mon[49304]: Health check update: Degraded data redundancy: 265/1707 objects degraded (15.524%), 35 pgs degraded (PG_DEGRADED) 2024-09-21T10:48:51.751 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:51 smithi123 ceph-mon[63856]: pgmap v304: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 346 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 4.1 MiB/s wr, 395 op/s; 264/1698 objects degraded (15.548%) 2024-09-21T10:48:51.846 INFO:teuthology.orchestra.run.smithi123.stdout: 5 268570 9.48 MB/sec execute 592 sec latency 42.295 ms 2024-09-21T10:48:51.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:51 smithi175 ceph-mon[49304]: pgmap v304: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 346 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 4.1 MiB/s wr, 395 op/s; 264/1698 objects degraded (15.548%) 2024-09-21T10:48:52.844 INFO:teuthology.orchestra.run.smithi123.stdout: 5 269039 9.48 MB/sec execute 593 sec latency 47.810 ms 2024-09-21T10:48:53.027 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:53 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:48:53.027 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:53 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:48:53.027 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:53 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:48:53.027 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:53 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:48:53.028 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:53 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:48:53.028 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:53 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:48:53.028 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:53 smithi123 ceph-mon[63856]: pgmap v305: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 347 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 5.3 MiB/s wr, 516 op/s; 258/1674 objects degraded (15.412%) 2024-09-21T10:48:53.028 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:53 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:48:53.028 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:53 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:48:53.028 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:53 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:48:53.028 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:53 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:48:53.028 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:53 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:48:53.029 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:53 smithi123 ceph-mon[63856]: Upgrade: Setting container_image for all osd 2024-09-21T10:48:53.029 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:53 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:48:53.029 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:53 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.0"} : dispatch 2024-09-21T10:48:53.029 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:53 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-09-21T10:48:53.029 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:53 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.1"} : dispatch 2024-09-21T10:48:53.029 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:53 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-09-21T10:48:53.029 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:53 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.2"} : dispatch 2024-09-21T10:48:53.029 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:53 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-09-21T10:48:53.029 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:53 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.3"} : dispatch 2024-09-21T10:48:53.030 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:53 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-09-21T10:48:53.030 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:53 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.4"} : dispatch 2024-09-21T10:48:53.030 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:53 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-09-21T10:48:53.030 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:53 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.5"} : dispatch 2024-09-21T10:48:53.030 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:53 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-09-21T10:48:53.030 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:53 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.6"} : dispatch 2024-09-21T10:48:53.030 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:53 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-09-21T10:48:53.030 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:53 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.7"} : dispatch 2024-09-21T10:48:53.030 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:53 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-09-21T10:48:53.030 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:53 smithi123 ceph-mon[63856]: Upgrade: Setting require_osd_release to 19 squid 2024-09-21T10:48:53.031 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:53 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd require-osd-release", "release": "squid"} : dispatch 2024-09-21T10:48:53.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:53 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:48:53.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:53 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:48:53.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:53 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:48:53.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:53 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:48:53.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:53 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:48:53.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:53 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:48:53.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:53 smithi175 ceph-mon[49304]: pgmap v305: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 347 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 5.3 MiB/s wr, 516 op/s; 258/1674 objects degraded (15.412%) 2024-09-21T10:48:53.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:53 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:48:53.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:53 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:48:53.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:53 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:48:53.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:53 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:48:53.375 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:53 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:48:53.375 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:53 smithi175 ceph-mon[49304]: Upgrade: Setting container_image for all osd 2024-09-21T10:48:53.375 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:53 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:48:53.375 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:53 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.0"} : dispatch 2024-09-21T10:48:53.375 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:53 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-09-21T10:48:53.375 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:53 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.1"} : dispatch 2024-09-21T10:48:53.375 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:53 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-09-21T10:48:53.375 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:53 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.2"} : dispatch 2024-09-21T10:48:53.375 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:53 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-09-21T10:48:53.375 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:53 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.3"} : dispatch 2024-09-21T10:48:53.375 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:53 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-09-21T10:48:53.375 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:53 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.4"} : dispatch 2024-09-21T10:48:53.375 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:53 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-09-21T10:48:53.375 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:53 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.5"} : dispatch 2024-09-21T10:48:53.376 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:53 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-09-21T10:48:53.376 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:53 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.6"} : dispatch 2024-09-21T10:48:53.376 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:53 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-09-21T10:48:53.376 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:53 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.7"} : dispatch 2024-09-21T10:48:53.376 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:53 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-09-21T10:48:53.376 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:53 smithi175 ceph-mon[49304]: Upgrade: Setting require_osd_release to 19 squid 2024-09-21T10:48:53.376 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:53 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd require-osd-release", "release": "squid"} : dispatch 2024-09-21T10:48:53.844 INFO:teuthology.orchestra.run.smithi123.stdout: 5 269609 9.49 MB/sec execute 594 sec latency 32.883 ms 2024-09-21T10:48:54.258 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:54 smithi123 ceph-mon[63856]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-09-21T10:48:54.258 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:54 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-09-21T10:48:54.258 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:54 smithi123 ceph-mon[63856]: osdmap e164: 8 total, 7 up, 8 in 2024-09-21T10:48:54.259 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:54 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "mds ok-to-stop", "ids": ["foofs.smithi123.uqoyow"]} : dispatch 2024-09-21T10:48:54.259 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:54 smithi123 ceph-mon[63856]: Upgrade: It appears safe to stop mds.foofs.smithi123.uqoyow 2024-09-21T10:48:54.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:54 smithi175 ceph-mon[49304]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-09-21T10:48:54.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:54 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-09-21T10:48:54.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:54 smithi175 ceph-mon[49304]: osdmap e164: 8 total, 7 up, 8 in 2024-09-21T10:48:54.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:54 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "mds ok-to-stop", "ids": ["foofs.smithi123.uqoyow"]} : dispatch 2024-09-21T10:48:54.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:54 smithi175 ceph-mon[49304]: Upgrade: It appears safe to stop mds.foofs.smithi123.uqoyow 2024-09-21T10:48:54.844 INFO:teuthology.orchestra.run.smithi123.stdout: 5 270047 9.49 MB/sec execute 595 sec latency 29.354 ms 2024-09-21T10:48:55.301 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:55 smithi123 ceph-mon[63856]: pgmap v307: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 350 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.7 MiB/s wr, 609 op/s; 258/1677 objects degraded (15.385%) 2024-09-21T10:48:55.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:55 smithi175 ceph-mon[49304]: pgmap v307: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 350 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.7 MiB/s wr, 609 op/s; 258/1677 objects degraded (15.385%) 2024-09-21T10:48:55.844 INFO:teuthology.orchestra.run.smithi123.stdout: 5 270435 9.50 MB/sec execute 596 sec latency 43.218 ms 2024-09-21T10:48:56.054 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:56 smithi123 ceph-mon[63856]: Health check update: Degraded data redundancy: 258/1677 objects degraded (15.385%), 35 pgs degraded (PG_DEGRADED) 2024-09-21T10:48:56.054 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:56 smithi123 ceph-mon[63856]: Upgrade: Updating mds.foofs.smithi123.uqoyow 2024-09-21T10:48:56.054 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:56 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:48:56.055 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:56 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi123.uqoyow", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-21T10:48:56.055 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:56 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:48:56.055 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:56 smithi123 ceph-mon[63856]: Deploying daemon mds.foofs.smithi123.uqoyow on smithi123 2024-09-21T10:48:56.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:56 smithi175 ceph-mon[49304]: Health check update: Degraded data redundancy: 258/1677 objects degraded (15.385%), 35 pgs degraded (PG_DEGRADED) 2024-09-21T10:48:56.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:56 smithi175 ceph-mon[49304]: Upgrade: Updating mds.foofs.smithi123.uqoyow 2024-09-21T10:48:56.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:56 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:48:56.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:56 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi123.uqoyow", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-21T10:48:56.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:56 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:48:56.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:56 smithi175 ceph-mon[49304]: Deploying daemon mds.foofs.smithi123.uqoyow on smithi123 2024-09-21T10:48:56.844 INFO:teuthology.orchestra.run.smithi123.stdout: 5 270853 9.50 MB/sec execute 597 sec latency 59.380 ms 2024-09-21T10:48:57.048 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:48:56 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:48:56.714+0000 7f3062f27640 -1 osd.7 0 read_superblock omap replica is missing. 2024-09-21T10:48:57.073 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:57 smithi123 ceph-mon[63856]: pgmap v308: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 354 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 6.8 MiB/s wr, 674 op/s; 266/1752 objects degraded (15.183%) 2024-09-21T10:48:57.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:57 smithi175 ceph-mon[49304]: pgmap v308: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 354 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 6.8 MiB/s wr, 674 op/s; 266/1752 objects degraded (15.183%) 2024-09-21T10:48:57.844 INFO:teuthology.orchestra.run.smithi123.stdout: 5 271211 9.49 MB/sec execute 598 sec latency 61.833 ms 2024-09-21T10:48:58.623 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:48:58 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:48:58.267+0000 7f3062f27640 -1 osd.7 161 log_to_monitors true 2024-09-21T10:48:58.844 INFO:teuthology.orchestra.run.smithi123.stdout: 5 271456 9.49 MB/sec execute 599 sec latency 77.230 ms 2024-09-21T10:48:58.861 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:58 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:48:58.861 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:58 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:48:58.861 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:58 smithi123 ceph-mon[63856]: from='osd.7 [v2:172.21.15.175:6824/3229469052,v1:172.21.15.175:6825/3229469052]' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-21T10:48:58.861 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:58 smithi123 ceph-mon[63856]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-21T10:48:59.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:58 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:48:59.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:58 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:48:59.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:58 smithi175 ceph-mon[49304]: from='osd.7 [v2:172.21.15.175:6824/3229469052,v1:172.21.15.175:6825/3229469052]' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-21T10:48:59.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:58 smithi175 ceph-mon[49304]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-21T10:48:59.600 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:59 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-mon-smithi123[63852]: 2024-09-21T10:48:59.199+0000 7f6086769640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-21T10:48:59.623 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:48:59 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:48:59.130+0000 7f3056cc9640 -1 osd.7 161 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T10:48:59.844 INFO:teuthology.orchestra.run.smithi123.stdout: 5 cleanup 600 sec 2024-09-21T10:49:00.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:59 smithi123 ceph-mon[63856]: pgmap v309: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 354 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 9.4 KiB/s rd, 6.8 MiB/s wr, 684 op/s; 266/1716 objects degraded (15.501%) 2024-09-21T10:49:00.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:59 smithi123 ceph-mon[63856]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-21T10:49:00.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:59 smithi123 ceph-mon[63856]: osdmap e165: 8 total, 7 up, 8 in 2024-09-21T10:49:00.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:59 smithi123 ceph-mon[63856]: from='osd.7 [v2:172.21.15.175:6824/3229469052,v1:172.21.15.175:6825/3229469052]' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi175", "root=default"]} : dispatch 2024-09-21T10:49:00.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:59 smithi123 ceph-mon[63856]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi175", "root=default"]} : dispatch 2024-09-21T10:49:00.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:59 smithi123 ceph-mon[63856]: from='osd.7 ' entity='osd.7' 2024-09-21T10:49:00.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:59 smithi123 ceph-mon[63856]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T10:49:00.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:59 smithi123 ceph-mon[63856]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-21T10:49:00.102 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:59 smithi123 ceph-mon[63856]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-21T10:49:00.102 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:59 smithi123 ceph-mon[63856]: osd.7 [v2:172.21.15.175:6824/3229469052,v1:172.21.15.175:6825/3229469052] boot 2024-09-21T10:49:00.102 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:59 smithi123 ceph-mon[63856]: osdmap e166: 8 total, 8 up, 8 in 2024-09-21T10:49:00.102 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:59 smithi123 ceph-mon[63856]: Standby daemon mds.foofs.smithi175.jrlevw assigned to filesystem foofs as rank 0 2024-09-21T10:49:00.102 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:59 smithi123 ceph-mon[63856]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-21T10:49:00.102 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:59 smithi123 ceph-mon[63856]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-21T10:49:00.102 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:59 smithi123 ceph-mon[63856]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-09-21T10:49:00.102 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:59 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-21T10:49:00.102 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:48:59 smithi123 ceph-mon[63856]: fsmap foofs:1/1 {0=foofs.smithi175.jrlevw=up:replay} 2024-09-21T10:49:00.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:59 smithi175 ceph-mon[49304]: pgmap v309: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 354 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 9.4 KiB/s rd, 6.8 MiB/s wr, 684 op/s; 266/1716 objects degraded (15.501%) 2024-09-21T10:49:00.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:59 smithi175 ceph-mon[49304]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-21T10:49:00.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:59 smithi175 ceph-mon[49304]: osdmap e165: 8 total, 7 up, 8 in 2024-09-21T10:49:00.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:59 smithi175 ceph-mon[49304]: from='osd.7 [v2:172.21.15.175:6824/3229469052,v1:172.21.15.175:6825/3229469052]' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi175", "root=default"]} : dispatch 2024-09-21T10:49:00.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:59 smithi175 ceph-mon[49304]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi175", "root=default"]} : dispatch 2024-09-21T10:49:00.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:59 smithi175 ceph-mon[49304]: from='osd.7 ' entity='osd.7' 2024-09-21T10:49:00.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:59 smithi175 ceph-mon[49304]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T10:49:00.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:59 smithi175 ceph-mon[49304]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-21T10:49:00.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:59 smithi175 ceph-mon[49304]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-21T10:49:00.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:59 smithi175 ceph-mon[49304]: osd.7 [v2:172.21.15.175:6824/3229469052,v1:172.21.15.175:6825/3229469052] boot 2024-09-21T10:49:00.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:59 smithi175 ceph-mon[49304]: osdmap e166: 8 total, 8 up, 8 in 2024-09-21T10:49:00.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:59 smithi175 ceph-mon[49304]: Standby daemon mds.foofs.smithi175.jrlevw assigned to filesystem foofs as rank 0 2024-09-21T10:49:00.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:59 smithi175 ceph-mon[49304]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-21T10:49:00.125 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:59 smithi175 ceph-mon[49304]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-21T10:49:00.125 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:59 smithi175 ceph-mon[49304]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-09-21T10:49:00.125 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:59 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-21T10:49:00.125 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:48:59 smithi175 ceph-mon[49304]: fsmap foofs:1/1 {0=foofs.smithi175.jrlevw=up:replay} 2024-09-21T10:49:00.844 INFO:teuthology.orchestra.run.smithi123.stdout: 5 cleanup 601 sec 2024-09-21T10:49:01.100 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:00 smithi123 ceph-mon[63856]: Health check update: Degraded data redundancy: 266/1716 objects degraded (15.501%), 35 pgs degraded (PG_DEGRADED) 2024-09-21T10:49:01.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:00 smithi123 ceph-mon[63856]: osdmap e167: 8 total, 8 up, 8 in 2024-09-21T10:49:01.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:00 smithi175 ceph-mon[49304]: Health check update: Degraded data redundancy: 266/1716 objects degraded (15.501%), 35 pgs degraded (PG_DEGRADED) 2024-09-21T10:49:01.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:00 smithi175 ceph-mon[49304]: osdmap e167: 8 total, 8 up, 8 in 2024-09-21T10:49:01.601 INFO:journalctl@ceph.osd.3.smithi123.stdout:Sep 21 10:49:01 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-3[76185]: 2024-09-21T10:49:01.330+0000 7ffbde740640 -1 bdev(0x55d4080dcc00 /var/lib/ceph/osd/ceph-3/block) aio_submit retries 1 2024-09-21T10:49:01.623 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:49:01 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-5[55349]: 2024-09-21T10:49:01.330+0000 7f254468a640 -1 bdev(0x55f3ca608c00 /var/lib/ceph/osd/ceph-5/block) aio_submit retries 2 2024-09-21T10:49:01.624 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:49:01 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-5[55349]: 2024-09-21T10:49:01.334+0000 7f2545e8d640 -1 bdev(0x55f3ca608c00 /var/lib/ceph/osd/ceph-5/block) aio_submit retries 5 2024-09-21T10:49:01.844 INFO:teuthology.orchestra.run.smithi123.stdout: 5 cleanup 602 sec 2024-09-21T10:49:02.100 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:01 smithi123 ceph-mon[63856]: pgmap v313: 97 pgs: 12 peering, 2 active+undersized, 28 active+undersized+degraded, 55 active+clean; 352 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 9.5 KiB/s rd, 6.1 MiB/s wr, 489 op/s; 199/1680 objects degraded (11.845%) 2024-09-21T10:49:02.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:01 smithi123 ceph-mon[63856]: osdmap e168: 8 total, 8 up, 8 in 2024-09-21T10:49:02.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:01 smithi175 ceph-mon[49304]: pgmap v313: 97 pgs: 12 peering, 2 active+undersized, 28 active+undersized+degraded, 55 active+clean; 352 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 9.5 KiB/s rd, 6.1 MiB/s wr, 489 op/s; 199/1680 objects degraded (11.845%) 2024-09-21T10:49:02.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:01 smithi175 ceph-mon[49304]: osdmap e168: 8 total, 8 up, 8 in 2024-09-21T10:49:02.844 INFO:teuthology.orchestra.run.smithi123.stdout: 5 cleanup 603 sec 2024-09-21T10:49:03.354 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:03 smithi123 ceph-mon[63856]: osdmap e169: 8 total, 8 up, 8 in 2024-09-21T10:49:03.354 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:03 smithi123 ceph-mon[63856]: pgmap v316: 97 pgs: 8 remapped+peering, 12 peering, 18 active+undersized+degraded, 59 active+clean; 358 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 278 B/s rd, 3.8 MiB/s wr, 458 op/s; 108/1683 objects degraded (6.417%) 2024-09-21T10:49:03.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:03 smithi175 ceph-mon[49304]: osdmap e169: 8 total, 8 up, 8 in 2024-09-21T10:49:03.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:03 smithi175 ceph-mon[49304]: pgmap v316: 97 pgs: 8 remapped+peering, 12 peering, 18 active+undersized+degraded, 59 active+clean; 358 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 278 B/s rd, 3.8 MiB/s wr, 458 op/s; 108/1683 objects degraded (6.417%) 2024-09-21T10:49:03.844 INFO:teuthology.orchestra.run.smithi123.stdout: 5 cleanup 604 sec 2024-09-21T10:49:04.350 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:04 smithi123 ceph-mon[63856]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-21T10:49:04.350 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:04 smithi123 ceph-mon[63856]: osdmap e170: 8 total, 8 up, 8 in 2024-09-21T10:49:04.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:04 smithi123 ceph-mon[63856]: mds.? [v2:172.21.15.123:6834/1493874291,v1:172.21.15.123:6835/1493874291] up:boot 2024-09-21T10:49:04.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:04 smithi123 ceph-mon[63856]: fsmap foofs:1/1 {0=foofs.smithi175.jrlevw=up:replay} 1 up:standby 2024-09-21T10:49:04.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:04 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "mds metadata", "who": "foofs.smithi123.uqoyow"} : dispatch 2024-09-21T10:49:04.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:04 smithi175 ceph-mon[49304]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-21T10:49:04.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:04 smithi175 ceph-mon[49304]: osdmap e170: 8 total, 8 up, 8 in 2024-09-21T10:49:04.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:04 smithi175 ceph-mon[49304]: mds.? [v2:172.21.15.123:6834/1493874291,v1:172.21.15.123:6835/1493874291] up:boot 2024-09-21T10:49:04.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:04 smithi175 ceph-mon[49304]: fsmap foofs:1/1 {0=foofs.smithi175.jrlevw=up:replay} 1 up:standby 2024-09-21T10:49:04.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:04 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "mds metadata", "who": "foofs.smithi123.uqoyow"} : dispatch 2024-09-21T10:49:04.845 INFO:teuthology.orchestra.run.smithi123.stdout: 5 cleanup 605 sec 2024-09-21T10:49:05.350 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:05 smithi123 ceph-mon[63856]: pgmap v318: 97 pgs: 6 remapped+peering, 12 peering, 79 active+clean; 353 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 2.8 MiB/s wr, 358 op/s; 2.5 MiB/s, 39 objects/s recovering 2024-09-21T10:49:05.350 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:05 smithi123 ceph-mon[63856]: Health check update: Degraded data redundancy: 108/1683 objects degraded (6.417%), 18 pgs degraded (PG_DEGRADED) 2024-09-21T10:49:05.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:05 smithi175 ceph-mon[49304]: pgmap v318: 97 pgs: 6 remapped+peering, 12 peering, 79 active+clean; 353 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 2.8 MiB/s wr, 358 op/s; 2.5 MiB/s, 39 objects/s recovering 2024-09-21T10:49:05.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:05 smithi175 ceph-mon[49304]: Health check update: Degraded data redundancy: 108/1683 objects degraded (6.417%), 18 pgs degraded (PG_DEGRADED) 2024-09-21T10:49:05.845 INFO:teuthology.orchestra.run.smithi123.stdout: 5 cleanup 606 sec 2024-09-21T10:49:06.336 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:06 smithi123 ceph-mon[63856]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 108/1683 objects degraded (6.417%), 18 pgs degraded) 2024-09-21T10:49:06.336 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:06 smithi123 ceph-mon[63856]: reconnect by client.24325 172.21.15.175:0/3979637707 after 0.000999987 2024-09-21T10:49:06.336 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:06 smithi123 ceph-mon[63856]: reconnect by client.14548 172.21.15.123:0/3382387850 after 0.000999987 2024-09-21T10:49:06.337 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:06 smithi123 ceph-mon[63856]: mds.? [v2:172.21.15.175:6832/3454320220,v1:172.21.15.175:6833/3454320220] up:reconnect 2024-09-21T10:49:06.337 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:06 smithi123 ceph-mon[63856]: fsmap foofs:1/1 {0=foofs.smithi175.jrlevw=up:reconnect} 1 up:standby 2024-09-21T10:49:06.337 INFO:teuthology.orchestra.run.smithi123.stdout:true 2024-09-21T10:49:06.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:06 smithi175 ceph-mon[49304]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 108/1683 objects degraded (6.417%), 18 pgs degraded) 2024-09-21T10:49:06.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:06 smithi175 ceph-mon[49304]: reconnect by client.24325 172.21.15.175:0/3979637707 after 0.000999987 2024-09-21T10:49:06.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:06 smithi175 ceph-mon[49304]: reconnect by client.14548 172.21.15.123:0/3382387850 after 0.000999987 2024-09-21T10:49:06.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:06 smithi175 ceph-mon[49304]: mds.? [v2:172.21.15.175:6832/3454320220,v1:172.21.15.175:6833/3454320220] up:reconnect 2024-09-21T10:49:06.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:06 smithi175 ceph-mon[49304]: fsmap foofs:1/1 {0=foofs.smithi175.jrlevw=up:reconnect} 1 up:standby 2024-09-21T10:49:06.844 INFO:teuthology.orchestra.run.smithi123.stdout: 5 cleanup 607 sec 2024-09-21T10:49:07.038 INFO:teuthology.orchestra.run.smithi123.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T10:49:07.038 INFO:teuthology.orchestra.run.smithi123.stdout:alertmanager.smithi123 smithi123 *:9093,9094 running (9m) 2m ago 18m 22.0M - 0.27.0 11f11916f8cd 5d550bd97de5 2024-09-21T10:49:07.038 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi123 smithi123 running (6m) 2m ago 18m 6899k - 19.3.0-5103-g5c1cdc3c 8f4d013db793 c5e1ef39a0a8 2024-09-21T10:49:07.038 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi175 smithi175 running (6m) 19s ago 18m 6895k - 19.3.0-5103-g5c1cdc3c 8f4d013db793 fb1ca999ccfd 2024-09-21T10:49:07.038 INFO:teuthology.orchestra.run.smithi123.stdout:grafana.smithi123 smithi123 *:3000 running (9m) 2m ago 18m 52.6M - dad864ee21e9 771df81763ba 2024-09-21T10:49:07.038 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi123.uqoyow smithi123 running (12m) 2m ago 12m 41.2M - 17.2.0 e1d6a67b021e f0a21eaac266 2024-09-21T10:49:07.038 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi175.jrlevw smithi175 running (12m) 19s ago 12m 14.7M - 17.2.0 e1d6a67b021e b21b16f4bc50 2024-09-21T10:49:07.039 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi123.stnuit smithi123 *:8443,9283,8765 running (8m) 2m ago 19m 519M - 19.3.0-5103-g5c1cdc3c 8f4d013db793 6812cc5cb56b 2024-09-21T10:49:07.039 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi175.txvwrq smithi175 *:8443,9283,8765 running (7m) 19s ago 18m 456M - 19.3.0-5103-g5c1cdc3c 8f4d013db793 304f0408075e 2024-09-21T10:49:07.039 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi123 smithi123 running (7m) 2m ago 20m 52.0M 2048M 19.3.0-5103-g5c1cdc3c 8f4d013db793 6b95207a2d80 2024-09-21T10:49:07.039 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi175 smithi175 running (7m) 19s ago 17m 46.8M 2048M 19.3.0-5103-g5c1cdc3c 8f4d013db793 3abee2d37f0f 2024-09-21T10:49:07.039 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.0.0.smithi123.ddatpr smithi123 *:2049 running (12m) 2m ago 12m 281M - 4.0 e1d6a67b021e 36f8cb2281eb 2024-09-21T10:49:07.039 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.1.0.smithi175.wpawer smithi175 *:2049 running (12m) 19s ago 12m 90.9M - 4.0 e1d6a67b021e 2200aeed9200 2024-09-21T10:49:07.039 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi123 smithi123 *:9100 running (9m) 2m ago 18m 12.3M - 1.7.0 72c9c2088986 ca7d0b29af81 2024-09-21T10:49:07.039 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi175 smithi175 *:9100 running (8m) 19s ago 17m 11.8M - 1.7.0 72c9c2088986 46c4f6b7c149 2024-09-21T10:49:07.039 INFO:teuthology.orchestra.run.smithi123.stdout:osd.0 smithi123 running (5m) 2m ago 17m 362M 1936M 19.3.0-5103-g5c1cdc3c 8f4d013db793 cc40f695c3a8 2024-09-21T10:49:07.039 INFO:teuthology.orchestra.run.smithi123.stdout:osd.1 smithi123 running (4m) 2m ago 16m 279M 1936M 19.3.0-5103-g5c1cdc3c 8f4d013db793 1be249025c7f 2024-09-21T10:49:07.039 INFO:teuthology.orchestra.run.smithi123.stdout:osd.2 smithi123 running (3m) 2m ago 16m 232M 1936M 19.3.0-5103-g5c1cdc3c 8f4d013db793 647aec759e3c 2024-09-21T10:49:07.039 INFO:teuthology.orchestra.run.smithi123.stdout:osd.3 smithi123 running (3m) 2m ago 15m 11.6M 1936M 19.3.0-5103-g5c1cdc3c 8f4d013db793 7a082fcad8e8 2024-09-21T10:49:07.040 INFO:teuthology.orchestra.run.smithi123.stdout:osd.4 smithi175 running (2m) 19s ago 15m 326M 2704M 19.3.0-5103-g5c1cdc3c 8f4d013db793 b650f72d10f0 2024-09-21T10:49:07.040 INFO:teuthology.orchestra.run.smithi123.stdout:osd.5 smithi175 running (100s) 19s ago 14m 311M 2704M 19.3.0-5103-g5c1cdc3c 8f4d013db793 8199a0b362df 2024-09-21T10:49:07.040 INFO:teuthology.orchestra.run.smithi123.stdout:osd.6 smithi175 running (61s) 19s ago 14m 263M 2704M 19.3.0-5103-g5c1cdc3c 8f4d013db793 aab90bc08918 2024-09-21T10:49:07.040 INFO:teuthology.orchestra.run.smithi123.stdout:osd.7 smithi175 running (23s) 19s ago 14m 12.1M 2704M 19.3.0-5103-g5c1cdc3c 8f4d013db793 2f37c8ff343f 2024-09-21T10:49:07.040 INFO:teuthology.orchestra.run.smithi123.stdout:prometheus.smithi123 smithi123 *:9095 running (7m) 2m ago 17m 55.4M - 2.51.0 1d3b7f56885b 37f944c9de88 2024-09-21T10:49:07.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:07 smithi123 ceph-mon[63856]: mds.? [v2:172.21.15.175:6832/3454320220,v1:172.21.15.175:6833/3454320220] up:rejoin 2024-09-21T10:49:07.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:07 smithi123 ceph-mon[63856]: fsmap foofs:1/1 {0=foofs.smithi175.jrlevw=up:rejoin} 1 up:standby 2024-09-21T10:49:07.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:07 smithi123 ceph-mon[63856]: pgmap v319: 97 pgs: 97 active+clean; 353 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 31 MiB/s rd, 2.0 MiB/s wr, 259 op/s; 4.4 MiB/s, 60 objects/s recovering 2024-09-21T10:49:07.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:07 smithi123 ceph-mon[63856]: from='client.34466 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:49:07.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:07 smithi123 ceph-mon[63856]: from='client.34470 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:49:07.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:07 smithi175 ceph-mon[49304]: mds.? [v2:172.21.15.175:6832/3454320220,v1:172.21.15.175:6833/3454320220] up:rejoin 2024-09-21T10:49:07.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:07 smithi175 ceph-mon[49304]: fsmap foofs:1/1 {0=foofs.smithi175.jrlevw=up:rejoin} 1 up:standby 2024-09-21T10:49:07.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:07 smithi175 ceph-mon[49304]: pgmap v319: 97 pgs: 97 active+clean; 353 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 31 MiB/s rd, 2.0 MiB/s wr, 259 op/s; 4.4 MiB/s, 60 objects/s recovering 2024-09-21T10:49:07.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:07 smithi175 ceph-mon[49304]: from='client.34466 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:49:07.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:07 smithi175 ceph-mon[49304]: from='client.34470 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:49:07.525 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-09-21T10:49:07.526 INFO:teuthology.orchestra.run.smithi123.stdout: "mon": { 2024-09-21T10:49:07.526 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev)": 2 2024-09-21T10:49:07.526 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:49:07.526 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr": { 2024-09-21T10:49:07.526 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev)": 2 2024-09-21T10:49:07.526 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:49:07.526 INFO:teuthology.orchestra.run.smithi123.stdout: "osd": { 2024-09-21T10:49:07.526 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev)": 8 2024-09-21T10:49:07.526 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:49:07.526 INFO:teuthology.orchestra.run.smithi123.stdout: "mds": { 2024-09-21T10:49:07.526 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T10:49:07.527 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:49:07.527 INFO:teuthology.orchestra.run.smithi123.stdout: "overall": { 2024-09-21T10:49:07.527 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-09-21T10:49:07.527 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev)": 12 2024-09-21T10:49:07.527 INFO:teuthology.orchestra.run.smithi123.stdout: } 2024-09-21T10:49:07.527 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-09-21T10:49:07.845 INFO:teuthology.orchestra.run.smithi123.stdout: 5 cleanup 608 sec 2024-09-21T10:49:07.880 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-09-21T10:49:07.881 INFO:teuthology.orchestra.run.smithi123.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472", 2024-09-21T10:49:07.881 INFO:teuthology.orchestra.run.smithi123.stdout: "in_progress": true, 2024-09-21T10:49:07.881 INFO:teuthology.orchestra.run.smithi123.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T10:49:07.881 INFO:teuthology.orchestra.run.smithi123.stdout: "services_complete": [ 2024-09-21T10:49:07.881 INFO:teuthology.orchestra.run.smithi123.stdout: "osd", 2024-09-21T10:49:07.881 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr", 2024-09-21T10:49:07.882 INFO:teuthology.orchestra.run.smithi123.stdout: "crash", 2024-09-21T10:49:07.882 INFO:teuthology.orchestra.run.smithi123.stdout: "mon" 2024-09-21T10:49:07.882 INFO:teuthology.orchestra.run.smithi123.stdout: ], 2024-09-21T10:49:07.882 INFO:teuthology.orchestra.run.smithi123.stdout: "progress": "14/23 daemons upgraded", 2024-09-21T10:49:07.882 INFO:teuthology.orchestra.run.smithi123.stdout: "message": "Currently upgrading mds daemons", 2024-09-21T10:49:07.882 INFO:teuthology.orchestra.run.smithi123.stdout: "is_paused": false 2024-09-21T10:49:07.883 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-09-21T10:49:08.137 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:08 smithi123 ceph-mon[63856]: from='client.34474 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:49:08.137 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:08 smithi123 ceph-mon[63856]: mds.? [v2:172.21.15.175:6832/3454320220,v1:172.21.15.175:6833/3454320220] up:clientreplay 2024-09-21T10:49:08.137 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:08 smithi123 ceph-mon[63856]: fsmap foofs:1/1 {0=foofs.smithi175.jrlevw=up:clientreplay} 1 up:standby 2024-09-21T10:49:08.137 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:08 smithi123 ceph-mon[63856]: daemon mds.foofs.smithi175.jrlevw is now active in filesystem foofs as rank 0 2024-09-21T10:49:08.138 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:08 smithi123 ceph-mon[63856]: from='client.? 172.21.15.123:0/253550839' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:49:08.343 INFO:teuthology.orchestra.run.smithi123.stdout:HEALTH_OK 2024-09-21T10:49:08.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:08 smithi175 ceph-mon[49304]: from='client.34474 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:49:08.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:08 smithi175 ceph-mon[49304]: mds.? [v2:172.21.15.175:6832/3454320220,v1:172.21.15.175:6833/3454320220] up:clientreplay 2024-09-21T10:49:08.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:08 smithi175 ceph-mon[49304]: fsmap foofs:1/1 {0=foofs.smithi175.jrlevw=up:clientreplay} 1 up:standby 2024-09-21T10:49:08.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:08 smithi175 ceph-mon[49304]: daemon mds.foofs.smithi175.jrlevw is now active in filesystem foofs as rank 0 2024-09-21T10:49:08.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:08 smithi175 ceph-mon[49304]: from='client.? 172.21.15.123:0/253550839' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:49:08.845 INFO:teuthology.orchestra.run.smithi123.stdout: 5 cleanup 609 sec 2024-09-21T10:49:09.010 INFO:teuthology.orchestra.run.smithi123.stdout: 0 cleanup 609 sec 2024-09-21T10:49:09.010 INFO:teuthology.orchestra.run.smithi123.stdout: 2024-09-21T10:49:09.010 INFO:teuthology.orchestra.run.smithi123.stdout: Operation Count AvgLat MaxLat 2024-09-21T10:49:09.010 INFO:teuthology.orchestra.run.smithi123.stdout: ---------------------------------------- 2024-09-21T10:49:09.010 INFO:teuthology.orchestra.run.smithi123.stdout: NTCreateX 182942 4.567 2171.731 2024-09-21T10:49:09.010 INFO:teuthology.orchestra.run.smithi123.stdout: Close 134273 0.130 17.976 2024-09-21T10:49:09.010 INFO:teuthology.orchestra.run.smithi123.stdout: Rename 7739 9.027 145.772 2024-09-21T10:49:09.011 INFO:teuthology.orchestra.run.smithi123.stdout: Unlink 37023 7.171 8896.853 2024-09-21T10:49:09.011 INFO:teuthology.orchestra.run.smithi123.stdout: Qpathinfo 165839 1.246 8896.627 2024-09-21T10:49:09.011 INFO:teuthology.orchestra.run.smithi123.stdout: Qfileinfo 28874 0.007 1.644 2024-09-21T10:49:09.011 INFO:teuthology.orchestra.run.smithi123.stdout: Qfsinfo 30368 0.595 26.745 2024-09-21T10:49:09.011 INFO:teuthology.orchestra.run.smithi123.stdout: Sfileinfo 14911 14.003 8897.538 2024-09-21T10:49:09.011 INFO:teuthology.orchestra.run.smithi123.stdout: Find 63992 0.924 29.358 2024-09-21T10:49:09.011 INFO:teuthology.orchestra.run.smithi123.stdout: WriteX 90265 14.994 8897.445 2024-09-21T10:49:09.011 INFO:teuthology.orchestra.run.smithi123.stdout: ReadX 286344 0.010 21.239 2024-09-21T10:49:09.011 INFO:teuthology.orchestra.run.smithi123.stdout: LockX 594 0.372 2.556 2024-09-21T10:49:09.011 INFO:teuthology.orchestra.run.smithi123.stdout: UnlockX 594 0.323 2.379 2024-09-21T10:49:09.011 INFO:teuthology.orchestra.run.smithi123.stdout: Flush 12822 0.002 0.033 2024-09-21T10:49:09.011 INFO:teuthology.orchestra.run.smithi123.stdout: 2024-09-21T10:49:09.011 INFO:teuthology.orchestra.run.smithi123.stdout:Throughput 9.48925 MB/sec 5 clients 5 procs max_latency=2334.569 ms 2024-09-21T10:49:09.013 DEBUG:teuthology.orchestra.run.smithi123:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'umount /mnt/foo' 2024-09-21T10:49:09.046 DEBUG:teuthology.orchestra.run.smithi123:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'while ! mount -t nfs $(hostname):/fake /mnt/foo ; do sleep 5 ; done' 2024-09-21T10:49:09.164 DEBUG:teuthology.orchestra.run.smithi123:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'cd /mnt/foo && dbench 5 -t 5' 2024-09-21T10:49:09.186 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:09 smithi123 ceph-mon[63856]: from='client.34482 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:49:09.186 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:09 smithi123 ceph-mon[63856]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-21T10:49:09.186 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:09 smithi123 ceph-mon[63856]: Cluster is now healthy 2024-09-21T10:49:09.186 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:09 smithi123 ceph-mon[63856]: mds.? [v2:172.21.15.175:6832/3454320220,v1:172.21.15.175:6833/3454320220] up:active 2024-09-21T10:49:09.186 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:09 smithi123 ceph-mon[63856]: fsmap foofs:1 {0=foofs.smithi175.jrlevw=up:active} 1 up:standby 2024-09-21T10:49:09.186 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:09 smithi123 ceph-mon[63856]: pgmap v320: 97 pgs: 97 active+clean; 352 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 32 MiB/s rd, 285 KiB/s wr, 28 op/s; 3.6 MiB/s, 50 objects/s recovering 2024-09-21T10:49:09.186 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:09 smithi123 ceph-mon[63856]: from='client.? 172.21.15.123:0/2761767592' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T10:49:09.188 INFO:teuthology.orchestra.run.smithi123.stdout:dbench version 4.00 - Copyright Andrew Tridgell 1999-2004 2024-09-21T10:49:09.188 INFO:teuthology.orchestra.run.smithi123.stdout: 2024-09-21T10:49:09.188 INFO:teuthology.orchestra.run.smithi123.stdout:Running for 5 seconds with load '/usr/share/dbench/client.txt' and minimum warmup 1 secs 2024-09-21T10:49:09.189 INFO:teuthology.orchestra.run.smithi123.stdout:3 of 5 processes prepared for launch 0 sec 2024-09-21T10:49:09.289 INFO:teuthology.orchestra.run.smithi123.stdout:5 of 5 processes prepared for launch 0 sec 2024-09-21T10:49:09.289 INFO:teuthology.orchestra.run.smithi123.stdout:releasing clients 2024-09-21T10:49:09.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:09 smithi175 ceph-mon[49304]: from='client.34482 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:49:09.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:09 smithi175 ceph-mon[49304]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-21T10:49:09.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:09 smithi175 ceph-mon[49304]: Cluster is now healthy 2024-09-21T10:49:09.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:09 smithi175 ceph-mon[49304]: mds.? [v2:172.21.15.175:6832/3454320220,v1:172.21.15.175:6833/3454320220] up:active 2024-09-21T10:49:09.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:09 smithi175 ceph-mon[49304]: fsmap foofs:1 {0=foofs.smithi175.jrlevw=up:active} 1 up:standby 2024-09-21T10:49:09.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:09 smithi175 ceph-mon[49304]: pgmap v320: 97 pgs: 97 active+clean; 352 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 32 MiB/s rd, 285 KiB/s wr, 28 op/s; 3.6 MiB/s, 50 objects/s recovering 2024-09-21T10:49:09.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:09 smithi175 ceph-mon[49304]: from='client.? 172.21.15.123:0/2761767592' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T10:49:11.289 INFO:teuthology.orchestra.run.smithi123.stdout: 5 209 12.22 MB/sec execute 1 sec latency 209.714 ms 2024-09-21T10:49:11.549 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:11 smithi123 ceph-mon[63856]: pgmap v321: 97 pgs: 97 active+clean; 353 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 MiB/s rd, 500 KiB/s wr, 41 op/s; 3.2 MiB/s, 44 objects/s recovering 2024-09-21T10:49:11.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:11 smithi175 ceph-mon[49304]: pgmap v321: 97 pgs: 97 active+clean; 353 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 MiB/s rd, 500 KiB/s wr, 41 op/s; 3.2 MiB/s, 44 objects/s recovering 2024-09-21T10:49:12.290 INFO:teuthology.orchestra.run.smithi123.stdout: 5 294 12.74 MB/sec execute 2 sec latency 243.423 ms 2024-09-21T10:49:12.625 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:12 smithi123 ceph-mon[63856]: daemon mds.foofs.smithi123.uqoyow restarted 2024-09-21T10:49:12.625 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:12 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:12.625 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:12 smithi123 ceph-mon[63856]: osdmap e171: 8 total, 8 up, 8 in 2024-09-21T10:49:12.625 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:12 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:12.625 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:12 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:49:12.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:12 smithi175 ceph-mon[49304]: daemon mds.foofs.smithi123.uqoyow restarted 2024-09-21T10:49:12.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:12 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:12.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:12 smithi175 ceph-mon[49304]: osdmap e171: 8 total, 8 up, 8 in 2024-09-21T10:49:12.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:12 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:12.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:12 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:49:13.289 INFO:teuthology.orchestra.run.smithi123.stdout: 5 377 12.74 MB/sec execute 3 sec latency 196.192 ms 2024-09-21T10:49:13.416 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:13 smithi123 ceph-mon[63856]: pgmap v323: 97 pgs: 97 active+clean; 350 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 1.7 MiB/s wr, 73 op/s; 2.8 MiB/s, 39 objects/s recovering 2024-09-21T10:49:13.416 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:13 smithi123 ceph-mon[63856]: mds.? [v2:172.21.15.123:6834/3508768736,v1:172.21.15.123:6835/3508768736] up:boot 2024-09-21T10:49:13.416 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:13 smithi123 ceph-mon[63856]: fsmap foofs:1 {0=foofs.smithi175.jrlevw=up:active} 1 up:standby 2024-09-21T10:49:13.416 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:13 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "mds metadata", "who": "foofs.smithi123.uqoyow"} : dispatch 2024-09-21T10:49:13.416 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:13 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:13.416 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:13 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:49:13.416 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:13 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:13.774 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:13 smithi175 ceph-mon[49304]: pgmap v323: 97 pgs: 97 active+clean; 350 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 1.7 MiB/s wr, 73 op/s; 2.8 MiB/s, 39 objects/s recovering 2024-09-21T10:49:13.774 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:13 smithi175 ceph-mon[49304]: mds.? [v2:172.21.15.123:6834/3508768736,v1:172.21.15.123:6835/3508768736] up:boot 2024-09-21T10:49:13.774 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:13 smithi175 ceph-mon[49304]: fsmap foofs:1 {0=foofs.smithi175.jrlevw=up:active} 1 up:standby 2024-09-21T10:49:13.774 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:13 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "mds metadata", "who": "foofs.smithi123.uqoyow"} : dispatch 2024-09-21T10:49:13.775 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:13 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:13.775 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:13 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:49:13.775 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:13 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:14.289 INFO:teuthology.orchestra.run.smithi123.stdout: 5 454 12.68 MB/sec execute 4 sec latency 139.845 ms 2024-09-21T10:49:15.290 INFO:teuthology.orchestra.run.smithi123.stdout: 5 cleanup 5 sec 2024-09-21T10:49:15.544 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:15 smithi123 ceph-mon[63856]: pgmap v324: 97 pgs: 97 active+clean; 335 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 3.8 MiB/s wr, 92 op/s; 1.5 MiB/s, 19 objects/s recovering 2024-09-21T10:49:15.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:15 smithi175 ceph-mon[49304]: pgmap v324: 97 pgs: 97 active+clean; 335 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 3.8 MiB/s wr, 92 op/s; 1.5 MiB/s, 19 objects/s recovering 2024-09-21T10:49:16.290 INFO:teuthology.orchestra.run.smithi123.stdout: 5 cleanup 6 sec 2024-09-21T10:49:16.344 INFO:teuthology.orchestra.run.smithi123.stdout: 0 cleanup 6 sec 2024-09-21T10:49:16.345 INFO:teuthology.orchestra.run.smithi123.stdout: 2024-09-21T10:49:16.345 INFO:teuthology.orchestra.run.smithi123.stdout: Operation Count AvgLat MaxLat 2024-09-21T10:49:16.345 INFO:teuthology.orchestra.run.smithi123.stdout: ---------------------------------------- 2024-09-21T10:49:16.345 INFO:teuthology.orchestra.run.smithi123.stdout: NTCreateX 337 37.798 139.840 2024-09-21T10:49:16.345 INFO:teuthology.orchestra.run.smithi123.stdout: Close 321 18.053 243.411 2024-09-21T10:49:16.345 INFO:teuthology.orchestra.run.smithi123.stdout: Unlink 5 9.480 15.258 2024-09-21T10:49:16.345 INFO:teuthology.orchestra.run.smithi123.stdout: Qpathinfo 22 0.916 3.971 2024-09-21T10:49:16.345 INFO:teuthology.orchestra.run.smithi123.stdout: Qfileinfo 316 18.949 66.470 2024-09-21T10:49:16.345 INFO:teuthology.orchestra.run.smithi123.stdout: Qfsinfo 36 0.571 2.877 2024-09-21T10:49:16.345 INFO:teuthology.orchestra.run.smithi123.stdout: Sfileinfo 15 16.450 33.750 2024-09-21T10:49:16.345 INFO:teuthology.orchestra.run.smithi123.stdout: Find 33 0.809 2.174 2024-09-21T10:49:16.345 INFO:teuthology.orchestra.run.smithi123.stdout: WriteX 1514 0.029 0.089 2024-09-21T10:49:16.346 INFO:teuthology.orchestra.run.smithi123.stdout: ReadX 123 0.023 0.984 2024-09-21T10:49:16.346 INFO:teuthology.orchestra.run.smithi123.stdout: Flush 18 17.553 41.148 2024-09-21T10:49:16.346 INFO:teuthology.orchestra.run.smithi123.stdout: 2024-09-21T10:49:16.346 INFO:teuthology.orchestra.run.smithi123.stdout:Throughput 12.6781 MB/sec 5 clients 5 procs max_latency=243.423 ms 2024-09-21T10:49:16.347 DEBUG:teuthology.parallel:result is None 2024-09-21T10:49:17.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:17 smithi123 ceph-mon[63856]: pgmap v325: 97 pgs: 97 active+clean; 326 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.2 MiB/s rd, 8.7 MiB/s wr, 243 op/s 2024-09-21T10:49:17.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:17 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:17.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:17 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:17.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:17 smithi175 ceph-mon[49304]: pgmap v325: 97 pgs: 97 active+clean; 326 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.2 MiB/s rd, 8.7 MiB/s wr, 243 op/s 2024-09-21T10:49:17.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:17 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:17.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:17 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:19.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:18 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:19.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:18 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:19.265 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:18 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:19.265 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:18 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:20.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:19 smithi123 ceph-mon[63856]: pgmap v326: 97 pgs: 97 active+clean; 327 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 861 KiB/s rd, 9.3 MiB/s wr, 245 op/s 2024-09-21T10:49:20.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:19 smithi175 ceph-mon[49304]: pgmap v326: 97 pgs: 97 active+clean; 327 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 861 KiB/s rd, 9.3 MiB/s wr, 245 op/s 2024-09-21T10:49:22.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:21 smithi175 ceph-mon[49304]: pgmap v327: 97 pgs: 97 active+clean; 280 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 11 MiB/s wr, 259 op/s 2024-09-21T10:49:22.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:21 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:22.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:21 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:22.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:21 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:49:22.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:21 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:49:22.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:21 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:22.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:21 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:22.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:21 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:49:22.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:21 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:49:22.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:21 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:49:22.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:21 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:49:22.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:21 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:49:22.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:21 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "mds ok-to-stop", "ids": ["foofs.smithi175.jrlevw"]} : dispatch 2024-09-21T10:49:22.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:21 smithi123 ceph-mon[63856]: pgmap v327: 97 pgs: 97 active+clean; 280 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 11 MiB/s wr, 259 op/s 2024-09-21T10:49:22.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:21 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:22.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:21 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:22.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:21 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:49:22.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:21 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:49:22.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:21 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:22.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:21 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:22.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:21 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:49:22.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:21 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:49:22.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:21 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:49:22.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:21 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:49:22.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:21 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:49:22.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:21 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "mds ok-to-stop", "ids": ["foofs.smithi175.jrlevw"]} : dispatch 2024-09-21T10:49:23.260 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:22 smithi175 ceph-mon[49304]: Detected new or changed devices on smithi123 2024-09-21T10:49:23.260 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:22 smithi175 ceph-mon[49304]: Upgrade: It appears safe to stop mds.foofs.smithi175.jrlevw 2024-09-21T10:49:23.350 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:22 smithi123 ceph-mon[63856]: Detected new or changed devices on smithi123 2024-09-21T10:49:23.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:22 smithi123 ceph-mon[63856]: Upgrade: It appears safe to stop mds.foofs.smithi175.jrlevw 2024-09-21T10:49:24.252 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:23 smithi175 ceph-mon[49304]: pgmap v328: 97 pgs: 97 active+clean; 250 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 9.1 MiB/s wr, 225 op/s 2024-09-21T10:49:24.253 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:23 smithi175 ceph-mon[49304]: Upgrade: Updating mds.foofs.smithi175.jrlevw 2024-09-21T10:49:24.253 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:23 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:24.253 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:23 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi175.jrlevw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-21T10:49:24.253 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:23 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:49:24.253 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:23 smithi175 ceph-mon[49304]: Deploying daemon mds.foofs.smithi175.jrlevw on smithi175 2024-09-21T10:49:24.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:23 smithi123 ceph-mon[63856]: pgmap v328: 97 pgs: 97 active+clean; 250 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 9.1 MiB/s wr, 225 op/s 2024-09-21T10:49:24.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:23 smithi123 ceph-mon[63856]: Upgrade: Updating mds.foofs.smithi175.jrlevw 2024-09-21T10:49:24.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:23 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:24.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:23 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi175.jrlevw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-21T10:49:24.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:23 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:49:24.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:23 smithi123 ceph-mon[63856]: Deploying daemon mds.foofs.smithi175.jrlevw on smithi175 2024-09-21T10:49:25.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:24 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-mon-smithi123[63852]: 2024-09-21T10:49:24.772+0000 7f6086769640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-21T10:49:25.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:24 smithi123 ceph-mon[63856]: pgmap v329: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 8.0 MiB/s wr, 202 op/s 2024-09-21T10:49:25.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:24 smithi123 ceph-mon[63856]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-21T10:49:25.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:24 smithi123 ceph-mon[63856]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-21T10:49:25.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:24 smithi123 ceph-mon[63856]: osdmap e172: 8 total, 8 up, 8 in 2024-09-21T10:49:25.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:24 smithi123 ceph-mon[63856]: Standby daemon mds.foofs.smithi123.uqoyow assigned to filesystem foofs as rank 0 2024-09-21T10:49:25.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:24 smithi123 ceph-mon[63856]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-21T10:49:25.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:24 smithi123 ceph-mon[63856]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-21T10:49:25.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:24 smithi123 ceph-mon[63856]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-09-21T10:49:25.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:24 smithi123 ceph-mon[63856]: fsmap foofs:1/1 {0=foofs.smithi123.uqoyow=up:replay} 2024-09-21T10:49:25.123 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:49:24 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-5[55349]: 2024-09-21T10:49:24.799+0000 7f2547690640 -1 bdev(0x55f3ca608c00 /var/lib/ceph/osd/ceph-5/block) aio_submit retries 5 2024-09-21T10:49:25.124 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:49:24 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6[57429]: 2024-09-21T10:49:24.791+0000 7fc54defc640 -1 bdev(0x55cb9676cc00 /var/lib/ceph/osd/ceph-6/block) aio_submit retries 1 2024-09-21T10:49:25.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:24 smithi175 ceph-mon[49304]: pgmap v329: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 8.0 MiB/s wr, 202 op/s 2024-09-21T10:49:25.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:24 smithi175 ceph-mon[49304]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-21T10:49:25.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:24 smithi175 ceph-mon[49304]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-21T10:49:25.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:24 smithi175 ceph-mon[49304]: osdmap e172: 8 total, 8 up, 8 in 2024-09-21T10:49:25.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:24 smithi175 ceph-mon[49304]: Standby daemon mds.foofs.smithi123.uqoyow assigned to filesystem foofs as rank 0 2024-09-21T10:49:25.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:24 smithi175 ceph-mon[49304]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-21T10:49:25.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:24 smithi175 ceph-mon[49304]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-21T10:49:25.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:24 smithi175 ceph-mon[49304]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-09-21T10:49:25.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:24 smithi175 ceph-mon[49304]: fsmap foofs:1/1 {0=foofs.smithi123.uqoyow=up:replay} 2024-09-21T10:49:27.350 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:26 smithi123 ceph-mon[63856]: reconnect by client.14548 172.21.15.123:0/3382387850 after 0.000999977 2024-09-21T10:49:27.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:26 smithi123 ceph-mon[63856]: reconnect by client.24325 172.21.15.175:0/3979637707 after 0.000999977 2024-09-21T10:49:27.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:26 smithi123 ceph-mon[63856]: mds.? [v2:172.21.15.123:6834/3508768736,v1:172.21.15.123:6835/3508768736] up:reconnect 2024-09-21T10:49:27.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:26 smithi123 ceph-mon[63856]: fsmap foofs:1/1 {0=foofs.smithi123.uqoyow=up:reconnect} 2024-09-21T10:49:27.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:26 smithi123 ceph-mon[63856]: pgmap v331: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 4.0 MiB/s rd, 2.3 MiB/s wr, 55 op/s 2024-09-21T10:49:27.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:26 smithi175 ceph-mon[49304]: reconnect by client.14548 172.21.15.123:0/3382387850 after 0.000999977 2024-09-21T10:49:27.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:26 smithi175 ceph-mon[49304]: reconnect by client.24325 172.21.15.175:0/3979637707 after 0.000999977 2024-09-21T10:49:27.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:26 smithi175 ceph-mon[49304]: mds.? [v2:172.21.15.123:6834/3508768736,v1:172.21.15.123:6835/3508768736] up:reconnect 2024-09-21T10:49:27.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:26 smithi175 ceph-mon[49304]: fsmap foofs:1/1 {0=foofs.smithi123.uqoyow=up:reconnect} 2024-09-21T10:49:27.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:26 smithi175 ceph-mon[49304]: pgmap v331: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 4.0 MiB/s rd, 2.3 MiB/s wr, 55 op/s 2024-09-21T10:49:28.350 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:27 smithi123 ceph-mon[63856]: mds.? [v2:172.21.15.123:6834/3508768736,v1:172.21.15.123:6835/3508768736] up:rejoin 2024-09-21T10:49:28.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:27 smithi123 ceph-mon[63856]: fsmap foofs:1/1 {0=foofs.smithi123.uqoyow=up:rejoin} 2024-09-21T10:49:28.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:27 smithi123 ceph-mon[63856]: daemon mds.foofs.smithi123.uqoyow is now active in filesystem foofs as rank 0 2024-09-21T10:49:28.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:27 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:28.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:27 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:49:28.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:27 smithi175 ceph-mon[49304]: mds.? [v2:172.21.15.123:6834/3508768736,v1:172.21.15.123:6835/3508768736] up:rejoin 2024-09-21T10:49:28.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:27 smithi175 ceph-mon[49304]: fsmap foofs:1/1 {0=foofs.smithi123.uqoyow=up:rejoin} 2024-09-21T10:49:28.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:27 smithi175 ceph-mon[49304]: daemon mds.foofs.smithi123.uqoyow is now active in filesystem foofs as rank 0 2024-09-21T10:49:28.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:27 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:28.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:27 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:49:29.350 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:28 smithi123 ceph-mon[63856]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-21T10:49:29.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:28 smithi123 ceph-mon[63856]: mds.? [v2:172.21.15.123:6834/3508768736,v1:172.21.15.123:6835/3508768736] up:active 2024-09-21T10:49:29.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:28 smithi123 ceph-mon[63856]: fsmap foofs:1 {0=foofs.smithi123.uqoyow=up:active} 2024-09-21T10:49:29.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:28 smithi123 ceph-mon[63856]: pgmap v332: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 1.6 MiB/s wr, 51 op/s 2024-09-21T10:49:29.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:28 smithi175 ceph-mon[49304]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-21T10:49:29.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:28 smithi175 ceph-mon[49304]: mds.? [v2:172.21.15.123:6834/3508768736,v1:172.21.15.123:6835/3508768736] up:active 2024-09-21T10:49:29.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:28 smithi175 ceph-mon[49304]: fsmap foofs:1 {0=foofs.smithi123.uqoyow=up:active} 2024-09-21T10:49:29.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:28 smithi175 ceph-mon[49304]: pgmap v332: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 1.6 MiB/s wr, 51 op/s 2024-09-21T10:49:31.850 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:31 smithi123 ceph-mon[63856]: pgmap v333: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 174 KiB/s wr, 30 op/s 2024-09-21T10:49:31.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:31 smithi175 ceph-mon[49304]: pgmap v333: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 174 KiB/s wr, 30 op/s 2024-09-21T10:49:33.682 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:33 smithi175 ceph-mon[49304]: pgmap v334: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 18 KiB/s wr, 16 op/s 2024-09-21T10:49:33.850 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:33 smithi123 ceph-mon[63856]: pgmap v334: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 18 KiB/s wr, 16 op/s 2024-09-21T10:49:35.691 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:35 smithi175 ceph-mon[49304]: pgmap v335: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 15 KiB/s wr, 11 op/s 2024-09-21T10:49:35.691 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:35 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:35.691 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:35 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:35.691 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:35 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:49:35.850 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:35 smithi123 ceph-mon[63856]: pgmap v335: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 15 KiB/s wr, 11 op/s 2024-09-21T10:49:35.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:35 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:35.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:35 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:35.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:35 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:49:36.667 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:36 smithi175 ceph-mon[49304]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-21T10:49:36.667 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:36 smithi175 ceph-mon[49304]: Cluster is now healthy 2024-09-21T10:49:36.667 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:36 smithi175 ceph-mon[49304]: mds.? [v2:172.21.15.175:6832/1763740386,v1:172.21.15.175:6833/1763740386] up:boot 2024-09-21T10:49:36.667 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:36 smithi175 ceph-mon[49304]: fsmap foofs:1 {0=foofs.smithi123.uqoyow=up:active} 1 up:standby 2024-09-21T10:49:36.667 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:36 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "mds metadata", "who": "foofs.smithi175.jrlevw"} : dispatch 2024-09-21T10:49:36.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:36 smithi123 ceph-mon[63856]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-21T10:49:36.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:36 smithi123 ceph-mon[63856]: Cluster is now healthy 2024-09-21T10:49:36.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:36 smithi123 ceph-mon[63856]: mds.? [v2:172.21.15.175:6832/1763740386,v1:172.21.15.175:6833/1763740386] up:boot 2024-09-21T10:49:36.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:36 smithi123 ceph-mon[63856]: fsmap foofs:1 {0=foofs.smithi123.uqoyow=up:active} 1 up:standby 2024-09-21T10:49:36.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:36 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "mds metadata", "who": "foofs.smithi175.jrlevw"} : dispatch 2024-09-21T10:49:37.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:37 smithi175 ceph-mon[49304]: pgmap v336: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 20 KiB/s wr, 13 op/s 2024-09-21T10:49:37.850 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:37 smithi123 ceph-mon[63856]: pgmap v336: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 20 KiB/s wr, 13 op/s 2024-09-21T10:49:38.715 INFO:teuthology.orchestra.run.smithi123.stdout:true 2024-09-21T10:49:39.065 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:38 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:39.066 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:38 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:39.066 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:38 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:39.066 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:38 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:39.100 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:38 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:39.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:38 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:39.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:38 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:39.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:38 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:39.411 INFO:teuthology.orchestra.run.smithi123.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T10:49:39.411 INFO:teuthology.orchestra.run.smithi123.stdout:alertmanager.smithi123 smithi123 *:9093,9094 running (10m) 22s ago 19m 22.1M - 0.27.0 11f11916f8cd 5d550bd97de5 2024-09-21T10:49:39.411 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi123 smithi123 running (6m) 22s ago 19m 6899k - 19.3.0-5103-g5c1cdc3c 8f4d013db793 c5e1ef39a0a8 2024-09-21T10:49:39.411 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi175 smithi175 running (6m) 1s ago 18m 6895k - 19.3.0-5103-g5c1cdc3c 8f4d013db793 fb1ca999ccfd 2024-09-21T10:49:39.411 INFO:teuthology.orchestra.run.smithi123.stdout:grafana.smithi123 smithi123 *:3000 running (9m) 22s ago 19m 52.2M - dad864ee21e9 771df81763ba 2024-09-21T10:49:39.411 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi123.uqoyow smithi123 running (28s) 22s ago 13m 11.8M - 19.3.0-5103-g5c1cdc3c 8f4d013db793 9b8c6a6dd421 2024-09-21T10:49:39.411 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi175.jrlevw smithi175 running (4s) 1s ago 13m 13.1M - 19.3.0-5103-g5c1cdc3c 8f4d013db793 052f5731455a 2024-09-21T10:49:39.411 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi123.stnuit smithi123 *:8443,9283,8765 running (9m) 22s ago 20m 523M - 19.3.0-5103-g5c1cdc3c 8f4d013db793 6812cc5cb56b 2024-09-21T10:49:39.411 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi175.txvwrq smithi175 *:8443,9283,8765 running (8m) 1s ago 18m 457M - 19.3.0-5103-g5c1cdc3c 8f4d013db793 304f0408075e 2024-09-21T10:49:39.411 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi123 smithi123 running (8m) 22s ago 20m 65.1M 2048M 19.3.0-5103-g5c1cdc3c 8f4d013db793 6b95207a2d80 2024-09-21T10:49:39.412 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi175 smithi175 running (7m) 1s ago 18m 48.9M 2048M 19.3.0-5103-g5c1cdc3c 8f4d013db793 3abee2d37f0f 2024-09-21T10:49:39.412 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.0.0.smithi123.ddatpr smithi123 *:2049 running (12m) 22s ago 12m 304M - 4.0 e1d6a67b021e 36f8cb2281eb 2024-09-21T10:49:39.412 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.1.0.smithi175.wpawer smithi175 *:2049 running (12m) 1s ago 12m 91.0M - 4.0 e1d6a67b021e 2200aeed9200 2024-09-21T10:49:39.412 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi123 smithi123 *:9100 running (9m) 22s ago 18m 12.1M - 1.7.0 72c9c2088986 ca7d0b29af81 2024-09-21T10:49:39.412 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi175 smithi175 *:9100 running (9m) 1s ago 18m 11.8M - 1.7.0 72c9c2088986 46c4f6b7c149 2024-09-21T10:49:39.412 INFO:teuthology.orchestra.run.smithi123.stdout:osd.0 smithi123 running (6m) 22s ago 17m 427M 1936M 19.3.0-5103-g5c1cdc3c 8f4d013db793 cc40f695c3a8 2024-09-21T10:49:39.412 INFO:teuthology.orchestra.run.smithi123.stdout:osd.1 smithi123 running (5m) 22s ago 17m 317M 1936M 19.3.0-5103-g5c1cdc3c 8f4d013db793 1be249025c7f 2024-09-21T10:49:39.412 INFO:teuthology.orchestra.run.smithi123.stdout:osd.2 smithi123 running (4m) 22s ago 16m 297M 1936M 19.3.0-5103-g5c1cdc3c 8f4d013db793 647aec759e3c 2024-09-21T10:49:39.412 INFO:teuthology.orchestra.run.smithi123.stdout:osd.3 smithi123 running (3m) 22s ago 16m 410M 1936M 19.3.0-5103-g5c1cdc3c 8f4d013db793 7a082fcad8e8 2024-09-21T10:49:39.412 INFO:teuthology.orchestra.run.smithi123.stdout:osd.4 smithi175 running (2m) 1s ago 15m 347M 2704M 19.3.0-5103-g5c1cdc3c 8f4d013db793 b650f72d10f0 2024-09-21T10:49:39.412 INFO:teuthology.orchestra.run.smithi123.stdout:osd.5 smithi175 running (2m) 1s ago 15m 338M 2704M 19.3.0-5103-g5c1cdc3c 8f4d013db793 8199a0b362df 2024-09-21T10:49:39.412 INFO:teuthology.orchestra.run.smithi123.stdout:osd.6 smithi175 running (93s) 1s ago 15m 313M 2704M 19.3.0-5103-g5c1cdc3c 8f4d013db793 aab90bc08918 2024-09-21T10:49:39.412 INFO:teuthology.orchestra.run.smithi123.stdout:osd.7 smithi175 running (55s) 1s ago 14m 304M 2704M 19.3.0-5103-g5c1cdc3c 8f4d013db793 2f37c8ff343f 2024-09-21T10:49:39.413 INFO:teuthology.orchestra.run.smithi123.stdout:prometheus.smithi123 smithi123 *:9095 running (8m) 22s ago 18m 58.0M - 2.51.0 1d3b7f56885b 37f944c9de88 2024-09-21T10:49:39.827 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-09-21T10:49:39.827 INFO:teuthology.orchestra.run.smithi123.stdout: "mon": { 2024-09-21T10:49:39.827 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev)": 2 2024-09-21T10:49:39.828 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:49:39.828 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr": { 2024-09-21T10:49:39.828 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev)": 2 2024-09-21T10:49:39.828 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:49:39.828 INFO:teuthology.orchestra.run.smithi123.stdout: "osd": { 2024-09-21T10:49:39.828 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev)": 8 2024-09-21T10:49:39.828 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:49:39.828 INFO:teuthology.orchestra.run.smithi123.stdout: "mds": { 2024-09-21T10:49:39.828 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev)": 2 2024-09-21T10:49:39.828 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:49:39.828 INFO:teuthology.orchestra.run.smithi123.stdout: "overall": { 2024-09-21T10:49:39.828 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev)": 14 2024-09-21T10:49:39.829 INFO:teuthology.orchestra.run.smithi123.stdout: } 2024-09-21T10:49:39.829 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-09-21T10:49:40.005 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:39 smithi175 ceph-mon[49304]: pgmap v337: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 7.2 KiB/s wr, 11 op/s 2024-09-21T10:49:40.006 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:39 smithi175 ceph-mon[49304]: from='client.34498 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:49:40.006 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:39 smithi175 ceph-mon[49304]: from='client.? 172.21.15.123:0/556088585' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:49:40.100 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:39 smithi123 ceph-mon[63856]: pgmap v337: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 7.2 KiB/s wr, 11 op/s 2024-09-21T10:49:40.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:39 smithi123 ceph-mon[63856]: from='client.34498 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:49:40.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:39 smithi123 ceph-mon[63856]: from='client.? 172.21.15.123:0/556088585' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:49:40.205 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-09-21T10:49:40.205 INFO:teuthology.orchestra.run.smithi123.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472", 2024-09-21T10:49:40.205 INFO:teuthology.orchestra.run.smithi123.stdout: "in_progress": true, 2024-09-21T10:49:40.205 INFO:teuthology.orchestra.run.smithi123.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T10:49:40.205 INFO:teuthology.orchestra.run.smithi123.stdout: "services_complete": [ 2024-09-21T10:49:40.206 INFO:teuthology.orchestra.run.smithi123.stdout: "osd", 2024-09-21T10:49:40.206 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr", 2024-09-21T10:49:40.206 INFO:teuthology.orchestra.run.smithi123.stdout: "mds", 2024-09-21T10:49:40.206 INFO:teuthology.orchestra.run.smithi123.stdout: "crash", 2024-09-21T10:49:40.206 INFO:teuthology.orchestra.run.smithi123.stdout: "mon" 2024-09-21T10:49:40.206 INFO:teuthology.orchestra.run.smithi123.stdout: ], 2024-09-21T10:49:40.206 INFO:teuthology.orchestra.run.smithi123.stdout: "progress": "16/23 daemons upgraded", 2024-09-21T10:49:40.206 INFO:teuthology.orchestra.run.smithi123.stdout: "message": "Currently upgrading mds daemons", 2024-09-21T10:49:40.206 INFO:teuthology.orchestra.run.smithi123.stdout: "is_paused": false 2024-09-21T10:49:40.206 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-09-21T10:49:40.626 INFO:teuthology.orchestra.run.smithi123.stdout:HEALTH_OK 2024-09-21T10:49:41.059 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:40 smithi175 ceph-mon[49304]: from='client.34502 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:49:41.059 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:40 smithi175 ceph-mon[49304]: from='client.34506 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:49:41.059 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:40 smithi175 ceph-mon[49304]: from='client.? 172.21.15.123:0/2925521757' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T10:49:41.100 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:40 smithi123 ceph-mon[63856]: from='client.34502 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:49:41.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:40 smithi123 ceph-mon[63856]: from='client.34506 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:49:41.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:40 smithi123 ceph-mon[63856]: from='client.? 172.21.15.123:0/2925521757' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T10:49:42.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:41 smithi175 ceph-mon[49304]: from='client.34514 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:49:42.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:41 smithi175 ceph-mon[49304]: pgmap v338: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 8.9 MiB/s rd, 7.2 KiB/s wr, 8 op/s 2024-09-21T10:49:42.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:41 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:42.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:41 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:42.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:41 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:49:42.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:41 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:49:42.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:41 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:42.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:41 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:42.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:41 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:49:42.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:41 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:49:42.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:41 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:49:42.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:41 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:49:42.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:41 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:49:42.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:41 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:49:42.262 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:41 smithi123 ceph-mon[63856]: from='client.34514 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:49:42.262 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:41 smithi123 ceph-mon[63856]: pgmap v338: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 8.9 MiB/s rd, 7.2 KiB/s wr, 8 op/s 2024-09-21T10:49:42.262 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:41 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:42.262 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:41 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:42.262 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:41 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:49:42.262 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:41 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:49:42.263 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:41 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:42.263 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:41 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:42.263 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:41 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:49:42.263 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:41 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:49:42.263 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:41 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:49:42.263 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:41 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:49:42.263 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:41 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:49:42.263 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:41 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:49:43.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:42 smithi175 ceph-mon[49304]: Detected new or changed devices on smithi175 2024-09-21T10:49:43.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:42 smithi175 ceph-mon[49304]: Upgrade: Setting container_image for all mds 2024-09-21T10:49:43.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:42 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:43.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:42 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi123.uqoyow"} : dispatch 2024-09-21T10:49:43.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:42 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi123.uqoyow"}]': finished 2024-09-21T10:49:43.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:42 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi175.jrlevw"} : dispatch 2024-09-21T10:49:43.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:42 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi175.jrlevw"}]': finished 2024-09-21T10:49:43.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:42 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:49:43.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:42 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:43.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:42 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:49:43.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:42 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:43.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:42 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:49:43.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:42 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:49:43.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:42 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:43.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:42 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:49:43.125 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:42 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:43.125 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:42 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:43.125 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:42 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:49:43.268 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:42 smithi123 ceph-mon[63856]: Detected new or changed devices on smithi175 2024-09-21T10:49:43.268 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:42 smithi123 ceph-mon[63856]: Upgrade: Setting container_image for all mds 2024-09-21T10:49:43.268 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:42 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:43.268 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:42 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi123.uqoyow"} : dispatch 2024-09-21T10:49:43.268 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:42 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi123.uqoyow"}]': finished 2024-09-21T10:49:43.268 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:42 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi175.jrlevw"} : dispatch 2024-09-21T10:49:43.269 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:42 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi175.jrlevw"}]': finished 2024-09-21T10:49:43.269 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:42 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:49:43.269 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:42 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:43.269 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:42 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:49:43.269 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:42 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:43.269 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:42 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:49:43.269 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:42 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:49:43.269 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:42 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:43.269 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:42 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:49:43.269 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:42 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:43.269 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:42 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:43.269 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:42 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:49:44.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:43 smithi175 ceph-mon[49304]: Upgrade: Setting container_image for all rgw 2024-09-21T10:49:44.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:43 smithi175 ceph-mon[49304]: Upgrade: Setting container_image for all rbd-mirror 2024-09-21T10:49:44.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:43 smithi175 ceph-mon[49304]: Upgrade: Setting container_image for all ceph-exporter 2024-09-21T10:49:44.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:43 smithi175 ceph-mon[49304]: Upgrade: Setting container_image for all iscsi 2024-09-21T10:49:44.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:43 smithi175 ceph-mon[49304]: pgmap v339: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 55 KiB/s rd, 7.2 KiB/s wr, 3 op/s 2024-09-21T10:49:44.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:43 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:44.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:43 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi123.ddatpr", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]} : dispatch 2024-09-21T10:49:44.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:43 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]} : dispatch 2024-09-21T10:49:44.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:43 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-21T10:49:44.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:43 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:49:44.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:43 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"} : dispatch 2024-09-21T10:49:44.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:43 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-21T10:49:44.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:43 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi123.ddatpr-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]} : dispatch 2024-09-21T10:49:44.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:43 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:49:44.225 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:43 smithi123 ceph-mon[63856]: Upgrade: Setting container_image for all rgw 2024-09-21T10:49:44.226 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:43 smithi123 ceph-mon[63856]: Upgrade: Setting container_image for all rbd-mirror 2024-09-21T10:49:44.226 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:43 smithi123 ceph-mon[63856]: Upgrade: Setting container_image for all ceph-exporter 2024-09-21T10:49:44.226 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:43 smithi123 ceph-mon[63856]: Upgrade: Setting container_image for all iscsi 2024-09-21T10:49:44.226 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:43 smithi123 ceph-mon[63856]: pgmap v339: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 55 KiB/s rd, 7.2 KiB/s wr, 3 op/s 2024-09-21T10:49:44.226 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:43 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:44.226 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:43 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi123.ddatpr", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]} : dispatch 2024-09-21T10:49:44.226 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:43 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]} : dispatch 2024-09-21T10:49:44.226 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:43 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-21T10:49:44.226 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:43 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:49:44.226 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:43 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"} : dispatch 2024-09-21T10:49:44.226 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:43 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-21T10:49:44.226 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:43 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi123.ddatpr-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]} : dispatch 2024-09-21T10:49:44.226 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:43 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:49:44.979 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:44 smithi123 ceph-mon[63856]: Upgrade: Updating nfs.foo.0.0.smithi123.ddatpr (1/2) 2024-09-21T10:49:44.979 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:44 smithi123 ceph-mon[63856]: Creating key for client.nfs.foo.0.0.smithi123.ddatpr 2024-09-21T10:49:44.979 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:44 smithi123 ceph-mon[63856]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-09-21T10:49:44.979 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:44 smithi123 ceph-mon[63856]: Rados config object exists: conf-nfs.foo 2024-09-21T10:49:44.979 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:44 smithi123 ceph-mon[63856]: Creating key for client.nfs.foo.0.0.smithi123.ddatpr-rgw 2024-09-21T10:49:44.979 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:44 smithi123 ceph-mon[63856]: Bind address in nfs.foo.0.0.smithi123.ddatpr's ganesha conf is defaulting to empty 2024-09-21T10:49:44.979 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:44 smithi123 ceph-mon[63856]: Deploying daemon nfs.foo.0.0.smithi123.ddatpr on smithi123 2024-09-21T10:49:45.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:44 smithi175 ceph-mon[49304]: Upgrade: Updating nfs.foo.0.0.smithi123.ddatpr (1/2) 2024-09-21T10:49:45.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:44 smithi175 ceph-mon[49304]: Creating key for client.nfs.foo.0.0.smithi123.ddatpr 2024-09-21T10:49:45.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:44 smithi175 ceph-mon[49304]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-09-21T10:49:45.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:44 smithi175 ceph-mon[49304]: Rados config object exists: conf-nfs.foo 2024-09-21T10:49:45.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:44 smithi175 ceph-mon[49304]: Creating key for client.nfs.foo.0.0.smithi123.ddatpr-rgw 2024-09-21T10:49:45.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:44 smithi175 ceph-mon[49304]: Bind address in nfs.foo.0.0.smithi123.ddatpr's ganesha conf is defaulting to empty 2024-09-21T10:49:45.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:44 smithi175 ceph-mon[49304]: Deploying daemon nfs.foo.0.0.smithi123.ddatpr on smithi123 2024-09-21T10:49:46.268 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:45 smithi123 ceph-mon[63856]: pgmap v340: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 682 B/s rd, 7.1 KiB/s wr, 3 op/s 2024-09-21T10:49:46.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:45 smithi175 ceph-mon[49304]: pgmap v340: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 682 B/s rd, 7.1 KiB/s wr, 3 op/s 2024-09-21T10:49:47.134 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:46 smithi123 ceph-mon[63856]: pgmap v341: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 7.3 KiB/s wr, 3 op/s 2024-09-21T10:49:47.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:46 smithi175 ceph-mon[49304]: pgmap v341: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 7.3 KiB/s wr, 3 op/s 2024-09-21T10:49:48.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:47 smithi175 ceph-mon[49304]: Evicting (and blocklisting) client session 14548 (172.21.15.123:0/3382387850) 2024-09-21T10:49:48.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:47 smithi175 ceph-mon[49304]: from='mds.? [v2:172.21.15.123:6834/3508768736,v1:172.21.15.123:6835/3508768736]' entity='mds.foofs.smithi123.uqoyow' cmd={"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.123:0/3382387850"} : dispatch 2024-09-21T10:49:48.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:47 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:48.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:47 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:48.350 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:47 smithi123 ceph-mon[63856]: Evicting (and blocklisting) client session 14548 (172.21.15.123:0/3382387850) 2024-09-21T10:49:48.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:47 smithi123 ceph-mon[63856]: from='mds.? [v2:172.21.15.123:6834/3508768736,v1:172.21.15.123:6835/3508768736]' entity='mds.foofs.smithi123.uqoyow' cmd={"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.123:0/3382387850"} : dispatch 2024-09-21T10:49:48.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:47 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:48.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:47 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:49.126 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:48 smithi175 ceph-mon[49304]: from='mds.? [v2:172.21.15.123:6834/3508768736,v1:172.21.15.123:6835/3508768736]' entity='mds.foofs.smithi123.uqoyow' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.123:0/3382387850"}]': finished 2024-09-21T10:49:49.126 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:48 smithi175 ceph-mon[49304]: osdmap e173: 8 total, 8 up, 8 in 2024-09-21T10:49:49.126 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:48 smithi175 ceph-mon[49304]: pgmap v343: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 614 B/s rd, 409 B/s wr, 0 op/s 2024-09-21T10:49:49.126 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:48 smithi175 ceph-mon[49304]: Upgrade: Updating nfs.foo.1.0.smithi175.wpawer (2/2) 2024-09-21T10:49:49.126 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:48 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:49.126 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:48 smithi175 ceph-mon[49304]: Creating key for client.nfs.foo.1.0.smithi175.wpawer 2024-09-21T10:49:49.126 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:48 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi175.wpawer", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]} : dispatch 2024-09-21T10:49:49.126 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:48 smithi175 ceph-mon[49304]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-09-21T10:49:49.126 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:48 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]} : dispatch 2024-09-21T10:49:49.126 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:48 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-21T10:49:49.127 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:48 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:49:49.127 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:48 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"} : dispatch 2024-09-21T10:49:49.127 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:48 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-21T10:49:49.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:48 smithi123 ceph-mon[63856]: from='mds.? [v2:172.21.15.123:6834/3508768736,v1:172.21.15.123:6835/3508768736]' entity='mds.foofs.smithi123.uqoyow' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.123:0/3382387850"}]': finished 2024-09-21T10:49:49.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:48 smithi123 ceph-mon[63856]: osdmap e173: 8 total, 8 up, 8 in 2024-09-21T10:49:49.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:48 smithi123 ceph-mon[63856]: pgmap v343: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 614 B/s rd, 409 B/s wr, 0 op/s 2024-09-21T10:49:49.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:48 smithi123 ceph-mon[63856]: Upgrade: Updating nfs.foo.1.0.smithi175.wpawer (2/2) 2024-09-21T10:49:49.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:48 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:49.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:48 smithi123 ceph-mon[63856]: Creating key for client.nfs.foo.1.0.smithi175.wpawer 2024-09-21T10:49:49.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:48 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi175.wpawer", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]} : dispatch 2024-09-21T10:49:49.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:48 smithi123 ceph-mon[63856]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-09-21T10:49:49.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:48 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]} : dispatch 2024-09-21T10:49:49.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:48 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-21T10:49:49.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:48 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:49:49.352 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:48 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"} : dispatch 2024-09-21T10:49:49.352 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:48 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-21T10:49:50.198 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:49 smithi175 ceph-mon[49304]: Rados config object exists: conf-nfs.foo 2024-09-21T10:49:50.198 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:49 smithi175 ceph-mon[49304]: Creating key for client.nfs.foo.1.0.smithi175.wpawer-rgw 2024-09-21T10:49:50.198 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:49 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi175.wpawer-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]} : dispatch 2024-09-21T10:49:50.198 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:49 smithi175 ceph-mon[49304]: Bind address in nfs.foo.1.0.smithi175.wpawer's ganesha conf is defaulting to empty 2024-09-21T10:49:50.198 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:49 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:49:50.198 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:49 smithi175 ceph-mon[49304]: Deploying daemon nfs.foo.1.0.smithi175.wpawer on smithi175 2024-09-21T10:49:50.225 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:49 smithi123 ceph-mon[63856]: Rados config object exists: conf-nfs.foo 2024-09-21T10:49:50.225 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:49 smithi123 ceph-mon[63856]: Creating key for client.nfs.foo.1.0.smithi175.wpawer-rgw 2024-09-21T10:49:50.225 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:49 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi175.wpawer-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]} : dispatch 2024-09-21T10:49:50.225 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:49 smithi123 ceph-mon[63856]: Bind address in nfs.foo.1.0.smithi175.wpawer's ganesha conf is defaulting to empty 2024-09-21T10:49:50.225 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:49 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:49:50.225 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:49 smithi123 ceph-mon[63856]: Deploying daemon nfs.foo.1.0.smithi175.wpawer on smithi175 2024-09-21T10:49:51.078 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:50 smithi175 ceph-mon[49304]: pgmap v344: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 614 B/s rd, 409 B/s wr, 0 op/s 2024-09-21T10:49:51.100 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:50 smithi123 ceph-mon[63856]: pgmap v344: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 614 B/s rd, 409 B/s wr, 0 op/s 2024-09-21T10:49:53.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:53 smithi175 ceph-mon[49304]: pgmap v345: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.8 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-09-21T10:49:53.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:53 smithi175 ceph-mon[49304]: Evicting (and blocklisting) client session 24325 (172.21.15.175:0/3979637707) 2024-09-21T10:49:53.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:53 smithi175 ceph-mon[49304]: from='mds.? [v2:172.21.15.123:6834/3508768736,v1:172.21.15.123:6835/3508768736]' entity='mds.foofs.smithi123.uqoyow' cmd={"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.175:0/3979637707"} : dispatch 2024-09-21T10:49:53.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:53 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:53.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:53 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:53.624 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:53 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:49:53.827 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:53 smithi123 ceph-mon[63856]: pgmap v345: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.8 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-09-21T10:49:53.827 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:53 smithi123 ceph-mon[63856]: Evicting (and blocklisting) client session 24325 (172.21.15.175:0/3979637707) 2024-09-21T10:49:53.827 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:53 smithi123 ceph-mon[63856]: from='mds.? [v2:172.21.15.123:6834/3508768736,v1:172.21.15.123:6835/3508768736]' entity='mds.foofs.smithi123.uqoyow' cmd={"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.175:0/3979637707"} : dispatch 2024-09-21T10:49:53.827 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:53 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:53.827 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:53 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:53.827 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:53 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:49:54.808 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:54 smithi175 ceph-mon[49304]: from='mds.? [v2:172.21.15.123:6834/3508768736,v1:172.21.15.123:6835/3508768736]' entity='mds.foofs.smithi123.uqoyow' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.175:0/3979637707"}]': finished 2024-09-21T10:49:54.808 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:54 smithi175 ceph-mon[49304]: osdmap e174: 8 total, 8 up, 8 in 2024-09-21T10:49:54.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:54 smithi123 ceph-mon[63856]: from='mds.? [v2:172.21.15.123:6834/3508768736,v1:172.21.15.123:6835/3508768736]' entity='mds.foofs.smithi123.uqoyow' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.175:0/3979637707"}]': finished 2024-09-21T10:49:54.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:54 smithi123 ceph-mon[63856]: osdmap e174: 8 total, 8 up, 8 in 2024-09-21T10:49:55.665 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:55 smithi123 ceph-mon[63856]: pgmap v347: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 3.1 KiB/s rd, 1.2 KiB/s wr, 4 op/s 2024-09-21T10:49:55.665 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:55 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:55.665 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:55 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:55.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:55 smithi175 ceph-mon[49304]: pgmap v347: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 3.1 KiB/s rd, 1.2 KiB/s wr, 4 op/s 2024-09-21T10:49:55.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:55 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:55.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:55 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:57.100 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:56 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:57.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:56 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:57.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:56 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:57.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:56 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:58.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:57 smithi175 ceph-mon[49304]: pgmap v348: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 4.5 KiB/s rd, 2.1 KiB/s wr, 7 op/s 2024-09-21T10:49:58.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:57 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:58.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:57 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:58.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:57 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:58.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:57 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:58.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:57 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:49:58.136 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:57 smithi123 ceph-mon[63856]: pgmap v348: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 4.5 KiB/s rd, 2.1 KiB/s wr, 7 op/s 2024-09-21T10:49:58.136 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:57 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:58.136 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:57 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:58.136 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:57 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:58.136 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:57 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:49:58.136 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:57 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:50:00.100 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:59 smithi123 ceph-mon[63856]: pgmap v349: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 3.8 KiB/s rd, 1.8 KiB/s wr, 6 op/s 2024-09-21T10:50:00.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:59 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:50:00.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:49:59 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:50:00.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:59 smithi175 ceph-mon[49304]: pgmap v349: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 3.8 KiB/s rd, 1.8 KiB/s wr, 6 op/s 2024-09-21T10:50:00.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:59 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:50:00.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:49:59 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:50:01.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:00 smithi123 ceph-mon[63856]: overall HEALTH_OK 2024-09-21T10:50:01.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:00 smithi175 ceph-mon[49304]: overall HEALTH_OK 2024-09-21T10:50:02.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:01 smithi175 ceph-mon[49304]: pgmap v350: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 3.8 KiB/s rd, 1.8 KiB/s wr, 6 op/s 2024-09-21T10:50:02.350 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:01 smithi123 ceph-mon[63856]: pgmap v350: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 3.8 KiB/s rd, 1.8 KiB/s wr, 6 op/s 2024-09-21T10:50:03.331 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:03 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:50:03.332 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:03 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:50:03.332 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:03 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:50:03.332 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:03 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:50:03.332 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:03 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:50:03.332 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:03 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:50:03.332 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:03 smithi123 ceph-mon[63856]: pgmap v351: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 1.6 KiB/s rd, 921 B/s wr, 2 op/s 2024-09-21T10:50:03.332 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:03 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:50:03.332 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:03 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:50:03.332 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:03 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:50:03.332 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:03 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:50:03.332 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:03 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:50:03.333 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:03 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:50:03.333 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:03 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:50:03.333 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:03 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:50:03.333 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:03 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:50:03.333 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:03 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:50:03.333 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:03 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:50:03.333 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:03 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:50:03.333 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:03 smithi123 ceph-mon[63856]: Upgrade: Setting container_image for all nfs 2024-09-21T10:50:03.333 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:03 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:50:03.333 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:03 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi123.ddatpr"} : dispatch 2024-09-21T10:50:03.333 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:03 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi123.ddatpr"}]': finished 2024-09-21T10:50:03.333 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:03 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi175.wpawer"} : dispatch 2024-09-21T10:50:03.333 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:03 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi175.wpawer"}]': finished 2024-09-21T10:50:03.333 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:03 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:50:03.333 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:03 smithi123 ceph-mon[63856]: Upgrade: Setting container_image for all nvmeof 2024-09-21T10:50:03.334 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:03 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:50:03.334 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:03 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:50:03.334 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:03 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:50:03.334 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:03 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:50:03.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:03 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:50:03.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:03 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:50:03.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:03 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:50:03.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:03 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:50:03.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:03 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:50:03.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:03 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:50:03.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:03 smithi175 ceph-mon[49304]: pgmap v351: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 1.6 KiB/s rd, 921 B/s wr, 2 op/s 2024-09-21T10:50:03.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:03 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:50:03.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:03 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:50:03.374 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:03 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:50:03.375 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:03 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:50:03.375 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:03 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:50:03.375 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:03 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:50:03.375 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:03 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:50:03.375 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:03 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:50:03.375 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:03 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:50:03.375 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:03 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:50:03.375 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:03 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:50:03.375 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:03 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:50:03.375 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:03 smithi175 ceph-mon[49304]: Upgrade: Setting container_image for all nfs 2024-09-21T10:50:03.375 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:03 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:50:03.375 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:03 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi123.ddatpr"} : dispatch 2024-09-21T10:50:03.375 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:03 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi123.ddatpr"}]': finished 2024-09-21T10:50:03.375 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:03 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi175.wpawer"} : dispatch 2024-09-21T10:50:03.376 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:03 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi175.wpawer"}]': finished 2024-09-21T10:50:03.376 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:03 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:50:03.376 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:03 smithi175 ceph-mon[49304]: Upgrade: Setting container_image for all nvmeof 2024-09-21T10:50:03.376 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:03 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:50:03.376 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:03 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:50:03.376 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:03 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:50:03.376 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:03 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:50:05.850 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:05 smithi123 ceph-mon[63856]: Upgrade: Updating grafana.smithi123 2024-09-21T10:50:05.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:05 smithi123 ceph-mon[63856]: pgmap v352: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 1.4 KiB/s rd, 841 B/s wr, 2 op/s 2024-09-21T10:50:05.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:05 smithi123 ceph-mon[63856]: Deploying daemon grafana.smithi123 on smithi123 2024-09-21T10:50:05.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:05 smithi175 ceph-mon[49304]: Upgrade: Updating grafana.smithi123 2024-09-21T10:50:05.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:05 smithi175 ceph-mon[49304]: pgmap v352: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 1.4 KiB/s rd, 841 B/s wr, 2 op/s 2024-09-21T10:50:05.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:05 smithi175 ceph-mon[49304]: Deploying daemon grafana.smithi123 on smithi123 2024-09-21T10:50:07.850 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:07 smithi123 ceph-mon[63856]: pgmap v353: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 1.9 KiB/s rd, 1.1 KiB/s wr, 3 op/s 2024-09-21T10:50:07.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:07 smithi175 ceph-mon[49304]: pgmap v353: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 1.9 KiB/s rd, 1.1 KiB/s wr, 3 op/s 2024-09-21T10:50:09.850 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:09 smithi123 ceph-mon[63856]: pgmap v354: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 682 B/s rd, 341 B/s wr, 1 op/s 2024-09-21T10:50:09.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:09 smithi175 ceph-mon[49304]: pgmap v354: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 682 B/s rd, 341 B/s wr, 1 op/s 2024-09-21T10:50:11.123 INFO:teuthology.orchestra.run.smithi123.stdout:true 2024-09-21T10:50:11.850 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:11 smithi123 ceph-mon[63856]: pgmap v355: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 682 B/s rd, 341 B/s wr, 1 op/s 2024-09-21T10:50:11.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:11 smithi175 ceph-mon[49304]: pgmap v355: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 682 B/s rd, 341 B/s wr, 1 op/s 2024-09-21T10:50:11.933 INFO:teuthology.orchestra.run.smithi123.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T10:50:11.934 INFO:teuthology.orchestra.run.smithi123.stdout:alertmanager.smithi123 smithi123 *:9093,9094 running (10m) 14s ago 19m 22.1M - 0.27.0 11f11916f8cd 5d550bd97de5 2024-09-21T10:50:11.934 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi123 smithi123 running (7m) 14s ago 19m 6899k - 19.3.0-5103-g5c1cdc3c 8f4d013db793 c5e1ef39a0a8 2024-09-21T10:50:11.934 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi175 smithi175 running (7m) 16s ago 19m 6895k - 19.3.0-5103-g5c1cdc3c 8f4d013db793 fb1ca999ccfd 2024-09-21T10:50:11.934 INFO:teuthology.orchestra.run.smithi123.stdout:grafana.smithi123 smithi123 *:3000 running (10m) 14s ago 19m 51.3M - dad864ee21e9 771df81763ba 2024-09-21T10:50:11.934 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi123.uqoyow smithi123 running (61s) 14s ago 13m 30.5M - 19.3.0-5103-g5c1cdc3c 8f4d013db793 9b8c6a6dd421 2024-09-21T10:50:11.934 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi175.jrlevw smithi175 running (37s) 16s ago 13m 13.2M - 19.3.0-5103-g5c1cdc3c 8f4d013db793 052f5731455a 2024-09-21T10:50:11.934 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi123.stnuit smithi123 *:8443,9283,8765 running (9m) 14s ago 21m 525M - 19.3.0-5103-g5c1cdc3c 8f4d013db793 6812cc5cb56b 2024-09-21T10:50:11.934 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi175.txvwrq smithi175 *:8443,9283,8765 running (8m) 16s ago 19m 457M - 19.3.0-5103-g5c1cdc3c 8f4d013db793 304f0408075e 2024-09-21T10:50:11.934 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi123 smithi123 running (8m) 14s ago 21m 64.3M 2048M 19.3.0-5103-g5c1cdc3c 8f4d013db793 6b95207a2d80 2024-09-21T10:50:11.934 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi175 smithi175 running (8m) 16s ago 19m 50.0M 2048M 19.3.0-5103-g5c1cdc3c 8f4d013db793 3abee2d37f0f 2024-09-21T10:50:11.934 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.0.0.smithi123.ddatpr smithi123 *:2049 running (25s) 14s ago 13m 64.3M - 5.9 8f4d013db793 8d4f82293026 2024-09-21T10:50:11.935 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.1.0.smithi175.wpawer smithi175 *:2049 running (19s) 16s ago 13m 24.8M - 5.9 8f4d013db793 86d9bf42d82a 2024-09-21T10:50:11.935 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi123 smithi123 *:9100 running (10m) 14s ago 19m 12.2M - 1.7.0 72c9c2088986 ca7d0b29af81 2024-09-21T10:50:11.935 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi175 smithi175 *:9100 running (9m) 16s ago 18m 12.2M - 1.7.0 72c9c2088986 46c4f6b7c149 2024-09-21T10:50:11.935 INFO:teuthology.orchestra.run.smithi123.stdout:osd.0 smithi123 running (6m) 14s ago 18m 437M 1936M 19.3.0-5103-g5c1cdc3c 8f4d013db793 cc40f695c3a8 2024-09-21T10:50:11.935 INFO:teuthology.orchestra.run.smithi123.stdout:osd.1 smithi123 running (5m) 14s ago 17m 320M 1936M 19.3.0-5103-g5c1cdc3c 8f4d013db793 1be249025c7f 2024-09-21T10:50:11.935 INFO:teuthology.orchestra.run.smithi123.stdout:osd.2 smithi123 running (4m) 14s ago 17m 299M 1936M 19.3.0-5103-g5c1cdc3c 8f4d013db793 647aec759e3c 2024-09-21T10:50:11.935 INFO:teuthology.orchestra.run.smithi123.stdout:osd.3 smithi123 running (4m) 14s ago 16m 414M 1936M 19.3.0-5103-g5c1cdc3c 8f4d013db793 7a082fcad8e8 2024-09-21T10:50:11.935 INFO:teuthology.orchestra.run.smithi123.stdout:osd.4 smithi175 running (3m) 16s ago 16m 348M 2704M 19.3.0-5103-g5c1cdc3c 8f4d013db793 b650f72d10f0 2024-09-21T10:50:11.935 INFO:teuthology.orchestra.run.smithi123.stdout:osd.5 smithi175 running (2m) 16s ago 16m 338M 2704M 19.3.0-5103-g5c1cdc3c 8f4d013db793 8199a0b362df 2024-09-21T10:50:11.935 INFO:teuthology.orchestra.run.smithi123.stdout:osd.6 smithi175 running (2m) 16s ago 15m 313M 2704M 19.3.0-5103-g5c1cdc3c 8f4d013db793 aab90bc08918 2024-09-21T10:50:11.935 INFO:teuthology.orchestra.run.smithi123.stdout:osd.7 smithi175 running (88s) 16s ago 15m 304M 2704M 19.3.0-5103-g5c1cdc3c 8f4d013db793 2f37c8ff343f 2024-09-21T10:50:11.935 INFO:teuthology.orchestra.run.smithi123.stdout:prometheus.smithi123 smithi123 *:9095 running (8m) 14s ago 18m 58.1M - 2.51.0 1d3b7f56885b 37f944c9de88 2024-09-21T10:50:12.366 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-09-21T10:50:12.367 INFO:teuthology.orchestra.run.smithi123.stdout: "mon": { 2024-09-21T10:50:12.367 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev)": 2 2024-09-21T10:50:12.367 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:50:12.367 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr": { 2024-09-21T10:50:12.367 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev)": 2 2024-09-21T10:50:12.367 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:50:12.367 INFO:teuthology.orchestra.run.smithi123.stdout: "osd": { 2024-09-21T10:50:12.367 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev)": 8 2024-09-21T10:50:12.367 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:50:12.367 INFO:teuthology.orchestra.run.smithi123.stdout: "mds": { 2024-09-21T10:50:12.367 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev)": 2 2024-09-21T10:50:12.367 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:50:12.368 INFO:teuthology.orchestra.run.smithi123.stdout: "overall": { 2024-09-21T10:50:12.368 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev)": 14 2024-09-21T10:50:12.368 INFO:teuthology.orchestra.run.smithi123.stdout: } 2024-09-21T10:50:12.368 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-09-21T10:50:12.618 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:12 smithi123 ceph-mon[63856]: from='client.34564 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:50:12.618 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:12 smithi123 ceph-mon[63856]: from='client.34568 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:50:12.618 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:12 smithi123 ceph-mon[63856]: from='client.? 172.21.15.123:0/1783446789' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:50:12.744 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-09-21T10:50:12.744 INFO:teuthology.orchestra.run.smithi123.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472", 2024-09-21T10:50:12.744 INFO:teuthology.orchestra.run.smithi123.stdout: "in_progress": true, 2024-09-21T10:50:12.744 INFO:teuthology.orchestra.run.smithi123.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T10:50:12.744 INFO:teuthology.orchestra.run.smithi123.stdout: "services_complete": [ 2024-09-21T10:50:12.744 INFO:teuthology.orchestra.run.smithi123.stdout: "osd", 2024-09-21T10:50:12.744 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr", 2024-09-21T10:50:12.744 INFO:teuthology.orchestra.run.smithi123.stdout: "nfs", 2024-09-21T10:50:12.745 INFO:teuthology.orchestra.run.smithi123.stdout: "mds", 2024-09-21T10:50:12.745 INFO:teuthology.orchestra.run.smithi123.stdout: "crash", 2024-09-21T10:50:12.745 INFO:teuthology.orchestra.run.smithi123.stdout: "mon" 2024-09-21T10:50:12.745 INFO:teuthology.orchestra.run.smithi123.stdout: ], 2024-09-21T10:50:12.745 INFO:teuthology.orchestra.run.smithi123.stdout: "progress": "18/23 daemons upgraded", 2024-09-21T10:50:12.745 INFO:teuthology.orchestra.run.smithi123.stdout: "message": "Currently upgrading grafana daemons", 2024-09-21T10:50:12.745 INFO:teuthology.orchestra.run.smithi123.stdout: "is_paused": false 2024-09-21T10:50:12.745 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-09-21T10:50:12.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:12 smithi175 ceph-mon[49304]: from='client.34564 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:50:12.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:12 smithi175 ceph-mon[49304]: from='client.34568 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:50:12.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:12 smithi175 ceph-mon[49304]: from='client.? 172.21.15.123:0/1783446789' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:50:13.189 INFO:teuthology.orchestra.run.smithi123.stdout:HEALTH_OK 2024-09-21T10:50:13.479 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:13 smithi123 ceph-mon[63856]: from='client.34572 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:50:13.479 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:13 smithi123 ceph-mon[63856]: pgmap v356: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 682 B/s rd, 341 B/s wr, 1 op/s 2024-09-21T10:50:13.479 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:13 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:50:13.479 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:13 smithi123 ceph-mon[63856]: from='client.34580 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:50:13.479 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:13 smithi123 ceph-mon[63856]: from='client.? 172.21.15.123:0/3921688818' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T10:50:13.775 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:13 smithi175 ceph-mon[49304]: from='client.34572 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:50:13.775 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:13 smithi175 ceph-mon[49304]: pgmap v356: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 682 B/s rd, 341 B/s wr, 1 op/s 2024-09-21T10:50:13.775 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:13 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:50:13.775 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:13 smithi175 ceph-mon[49304]: from='client.34580 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:50:13.775 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:13 smithi175 ceph-mon[49304]: from='client.? 172.21.15.123:0/3921688818' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T10:50:15.696 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:15 smithi123 ceph-mon[63856]: pgmap v357: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 682 B/s rd, 341 B/s wr, 1 op/s 2024-09-21T10:50:15.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:15 smithi175 ceph-mon[49304]: pgmap v357: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 682 B/s rd, 341 B/s wr, 1 op/s 2024-09-21T10:50:17.627 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:17 smithi123 ceph-mon[63856]: pgmap v358: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 852 B/s rd, 341 B/s wr, 1 op/s 2024-09-21T10:50:17.627 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:17 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:50:17.628 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:17 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:50:17.628 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:17 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:50:17.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:17 smithi175 ceph-mon[49304]: pgmap v358: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 852 B/s rd, 341 B/s wr, 1 op/s 2024-09-21T10:50:17.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:17 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:50:17.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:17 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:50:17.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:17 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:50:19.703 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:19 smithi123 ceph-mon[63856]: pgmap v359: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:50:19.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:19 smithi175 ceph-mon[49304]: pgmap v359: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:50:21.850 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:21 smithi123 ceph-mon[63856]: pgmap v360: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:50:21.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:21 smithi175 ceph-mon[49304]: pgmap v360: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:50:23.612 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:23 smithi123 ceph-mon[63856]: pgmap v361: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:50:23.612 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:23 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:50:23.613 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:23 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:50:23.613 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:23 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:50:23.613 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:23 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:50:23.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:23 smithi175 ceph-mon[49304]: pgmap v361: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:50:23.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:23 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:50:23.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:23 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:50:23.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:23 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:50:23.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:23 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:50:25.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:25 smithi175 ceph-mon[49304]: pgmap v362: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:50:25.628 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:25 smithi123 ceph-mon[63856]: pgmap v362: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:50:27.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:27 smithi175 ceph-mon[49304]: pgmap v363: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T10:50:27.850 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:27 smithi123 ceph-mon[63856]: pgmap v363: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T10:50:28.600 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:28 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:50:28.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:28 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:50:30.102 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:29 smithi123 ceph-mon[63856]: pgmap v364: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:50:30.102 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:29 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:50:30.102 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:29 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:50:30.102 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:29 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:50:30.102 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:29 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:50:30.102 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:29 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:50:30.102 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:29 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:50:30.102 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:29 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "dashboard get-grafana-api-url"} : dispatch 2024-09-21T10:50:30.102 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:29 smithi123 ceph-mon[63856]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T10:50:30.103 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:29 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:50:30.103 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:29 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:50:30.103 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:29 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:50:30.103 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:29 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:50:30.103 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:29 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:50:30.103 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:29 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:50:30.103 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:29 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:50:30.103 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:29 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:50:30.103 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:29 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:50:30.103 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:29 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:50:30.103 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:29 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:50:30.103 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:29 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:50:30.103 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:29 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:50:30.104 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:29 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:50:30.104 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:29 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:50:30.104 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:29 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:50:30.104 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:29 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:50:30.104 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:29 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:50:30.104 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:29 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:50:30.104 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:29 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:50:30.104 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:29 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-21T10:50:30.104 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:29 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-21T10:50:30.104 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:29 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:50:30.104 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:29 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-21T10:50:30.104 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:29 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-21T10:50:30.104 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:29 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-21T10:50:30.105 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:29 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-21T10:50:30.105 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:29 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-21T10:50:30.105 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:29 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-21T10:50:30.105 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:29 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-21T10:50:30.105 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:29 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-21T10:50:30.105 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:29 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-21T10:50:30.105 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:29 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-21T10:50:30.105 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:29 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-21T10:50:30.105 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:29 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:50:30.105 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:29 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-21T10:50:30.105 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:29 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-21T10:50:30.105 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:29 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-21T10:50:30.105 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:29 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-21T10:50:30.105 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:29 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-21T10:50:30.106 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:29 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-21T10:50:30.106 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:29 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-21T10:50:30.106 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:29 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-21T10:50:30.106 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:29 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:50:30.106 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:29 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:50:30.106 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:29 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:50:30.106 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:29 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:50:30.106 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:29 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:50:30.106 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:29 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:50:30.106 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:29 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-21T10:50:30.106 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:29 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-21T10:50:30.106 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:29 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:50:30.106 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:29 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:50:30.107 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:29 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:50:30.107 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:29 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:50:30.107 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:29 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:50:30.107 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:29 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:50:30.107 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:29 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:50:30.107 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:29 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:50:30.107 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:29 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:50:30.107 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:29 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:50:30.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:29 smithi175 ceph-mon[49304]: pgmap v364: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:50:30.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:29 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:50:30.125 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:29 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:50:30.125 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:29 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:50:30.125 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:29 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:50:30.125 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:29 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:50:30.125 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:29 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:50:30.125 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:29 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "dashboard get-grafana-api-url"} : dispatch 2024-09-21T10:50:30.126 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:29 smithi175 ceph-mon[49304]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T10:50:30.126 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:29 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:50:30.126 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:29 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:50:30.126 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:29 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:50:30.126 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:29 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:50:30.126 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:29 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:50:30.127 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:29 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:50:30.127 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:29 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:50:30.127 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:29 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:50:30.127 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:29 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:50:30.127 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:29 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:50:30.128 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:29 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:50:30.128 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:29 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:50:30.128 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:29 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:50:30.128 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:29 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:50:30.128 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:29 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:50:30.129 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:29 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:50:30.129 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:29 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:50:30.129 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:29 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:50:30.129 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:29 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:50:30.129 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:29 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:50:30.129 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:29 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-21T10:50:30.130 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:29 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-21T10:50:30.130 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:29 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:50:30.130 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:29 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-21T10:50:30.130 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:29 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-21T10:50:30.130 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:29 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-21T10:50:30.130 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:29 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-21T10:50:30.130 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:29 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-21T10:50:30.131 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:29 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-21T10:50:30.131 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:29 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-21T10:50:30.131 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:29 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-21T10:50:30.131 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:29 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-21T10:50:30.131 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:29 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-21T10:50:30.131 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:29 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-21T10:50:30.132 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:29 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:50:30.132 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:29 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-21T10:50:30.132 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:29 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-21T10:50:30.132 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:29 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-21T10:50:30.132 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:29 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-21T10:50:30.133 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:29 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-21T10:50:30.133 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:29 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-21T10:50:30.133 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:29 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-21T10:50:30.133 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:29 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-21T10:50:30.133 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:29 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:50:30.133 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:29 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:50:30.134 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:29 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:50:30.134 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:29 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:50:30.134 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:29 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:50:30.134 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:29 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:50:30.134 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:29 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-21T10:50:30.134 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:29 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-21T10:50:30.135 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:29 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:50:30.135 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:29 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:50:30.135 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:29 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:50:30.135 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:29 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:50:30.135 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:29 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:50:30.135 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:29 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:50:30.135 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:29 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:50:30.136 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:29 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:50:30.136 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:29 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:50:30.136 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:29 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:50:31.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:30 smithi123 ceph-mon[63856]: Upgrade: Finalizing container_image settings 2024-09-21T10:50:31.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:30 smithi123 ceph-mon[63856]: Upgrade: Complete! 2024-09-21T10:50:31.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:30 smithi175 ceph-mon[49304]: Upgrade: Finalizing container_image settings 2024-09-21T10:50:31.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:30 smithi175 ceph-mon[49304]: Upgrade: Complete! 2024-09-21T10:50:32.100 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:31 smithi123 ceph-mon[63856]: pgmap v365: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:50:32.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:31 smithi175 ceph-mon[49304]: pgmap v365: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:50:34.100 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:33 smithi123 ceph-mon[63856]: pgmap v366: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:50:34.100 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:33 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:50:34.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:33 smithi175 ceph-mon[49304]: pgmap v366: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:50:34.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:33 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:50:36.100 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:35 smithi123 ceph-mon[63856]: pgmap v367: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:50:36.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:35 smithi175 ceph-mon[49304]: pgmap v367: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:50:38.100 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:37 smithi123 ceph-mon[63856]: pgmap v368: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T10:50:38.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:37 smithi175 ceph-mon[49304]: pgmap v368: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T10:50:40.100 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:39 smithi123 ceph-mon[63856]: pgmap v369: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:50:40.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:39 smithi175 ceph-mon[49304]: pgmap v369: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:50:42.100 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:41 smithi123 ceph-mon[63856]: pgmap v370: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:50:42.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:41 smithi175 ceph-mon[49304]: pgmap v370: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:50:43.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:42 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:50:43.350 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:42 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:50:44.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:43 smithi175 ceph-mon[49304]: pgmap v371: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:50:44.201 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:43 smithi123 ceph-mon[63856]: pgmap v371: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:50:44.225 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -e sha1=5c1cdc3c470dae78e91a0ed2bcf13509a046d301 -- bash -c 'ceph orch ps' 2024-09-21T10:50:45.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:44 smithi123 ceph-mon[63856]: from='client.34588 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:50:45.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:44 smithi175 ceph-mon[49304]: from='client.34588 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:50:45.924 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:45 smithi123 ceph-mon[63856]: pgmap v372: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:50:46.367 INFO:teuthology.orchestra.run.smithi123.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T10:50:46.367 INFO:teuthology.orchestra.run.smithi123.stdout:alertmanager.smithi123 smithi123 *:9093,9094 running (11m) 24s ago 20m 22.1M - 0.27.0 11f11916f8cd 5d550bd97de5 2024-09-21T10:50:46.367 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi123 smithi123 running (7m) 24s ago 20m 6899k - 19.3.0-5103-g5c1cdc3c 8f4d013db793 c5e1ef39a0a8 2024-09-21T10:50:46.368 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi175 smithi175 running (7m) 51s ago 19m 6895k - 19.3.0-5103-g5c1cdc3c 8f4d013db793 fb1ca999ccfd 2024-09-21T10:50:46.368 INFO:teuthology.orchestra.run.smithi123.stdout:grafana.smithi123 smithi123 *:3000 running (29s) 24s ago 20m 36.9M - 10.4.8 5aad1d7cf004 af3591d01f0d 2024-09-21T10:50:46.368 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi123.uqoyow smithi123 running (95s) 24s ago 14m 30.7M - 19.3.0-5103-g5c1cdc3c 8f4d013db793 9b8c6a6dd421 2024-09-21T10:50:46.368 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi175.jrlevw smithi175 running (71s) 51s ago 14m 13.2M - 19.3.0-5103-g5c1cdc3c 8f4d013db793 052f5731455a 2024-09-21T10:50:46.368 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi123.stnuit smithi123 *:8443,9283,8765 running (10m) 24s ago 21m 533M - 19.3.0-5103-g5c1cdc3c 8f4d013db793 6812cc5cb56b 2024-09-21T10:50:46.368 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi175.txvwrq smithi175 *:8443,9283,8765 running (9m) 51s ago 19m 457M - 19.3.0-5103-g5c1cdc3c 8f4d013db793 304f0408075e 2024-09-21T10:50:46.368 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi123 smithi123 running (9m) 24s ago 21m 65.6M 2048M 19.3.0-5103-g5c1cdc3c 8f4d013db793 6b95207a2d80 2024-09-21T10:50:46.368 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi175 smithi175 running (8m) 51s ago 19m 50.0M 2048M 19.3.0-5103-g5c1cdc3c 8f4d013db793 3abee2d37f0f 2024-09-21T10:50:46.368 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.0.0.smithi123.ddatpr smithi123 *:2049 running (59s) 24s ago 13m 65.1M - 5.9 8f4d013db793 8d4f82293026 2024-09-21T10:50:46.368 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.1.0.smithi175.wpawer smithi175 *:2049 running (54s) 51s ago 13m 24.8M - 5.9 8f4d013db793 86d9bf42d82a 2024-09-21T10:50:46.368 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi123 smithi123 *:9100 running (10m) 24s ago 20m 12.2M - 1.7.0 72c9c2088986 ca7d0b29af81 2024-09-21T10:50:46.368 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi175 smithi175 *:9100 running (10m) 51s ago 19m 12.2M - 1.7.0 72c9c2088986 46c4f6b7c149 2024-09-21T10:50:46.369 INFO:teuthology.orchestra.run.smithi123.stdout:osd.0 smithi123 running (7m) 24s ago 18m 438M 1936M 19.3.0-5103-g5c1cdc3c 8f4d013db793 cc40f695c3a8 2024-09-21T10:50:46.369 INFO:teuthology.orchestra.run.smithi123.stdout:osd.1 smithi123 running (6m) 24s ago 18m 321M 1936M 19.3.0-5103-g5c1cdc3c 8f4d013db793 1be249025c7f 2024-09-21T10:50:46.369 INFO:teuthology.orchestra.run.smithi123.stdout:osd.2 smithi123 running (5m) 24s ago 17m 300M 1936M 19.3.0-5103-g5c1cdc3c 8f4d013db793 647aec759e3c 2024-09-21T10:50:46.369 INFO:teuthology.orchestra.run.smithi123.stdout:osd.3 smithi123 running (4m) 24s ago 17m 415M 1936M 19.3.0-5103-g5c1cdc3c 8f4d013db793 7a082fcad8e8 2024-09-21T10:50:46.369 INFO:teuthology.orchestra.run.smithi123.stdout:osd.4 smithi175 running (3m) 51s ago 17m 348M 2704M 19.3.0-5103-g5c1cdc3c 8f4d013db793 b650f72d10f0 2024-09-21T10:50:46.369 INFO:teuthology.orchestra.run.smithi123.stdout:osd.5 smithi175 running (3m) 51s ago 16m 338M 2704M 19.3.0-5103-g5c1cdc3c 8f4d013db793 8199a0b362df 2024-09-21T10:50:46.369 INFO:teuthology.orchestra.run.smithi123.stdout:osd.6 smithi175 running (2m) 51s ago 16m 313M 2704M 19.3.0-5103-g5c1cdc3c 8f4d013db793 aab90bc08918 2024-09-21T10:50:46.369 INFO:teuthology.orchestra.run.smithi123.stdout:osd.7 smithi175 running (2m) 51s ago 15m 304M 2704M 19.3.0-5103-g5c1cdc3c 8f4d013db793 2f37c8ff343f 2024-09-21T10:50:46.369 INFO:teuthology.orchestra.run.smithi123.stdout:prometheus.smithi123 smithi123 *:9095 running (9m) 24s ago 19m 58.1M - 2.51.0 1d3b7f56885b 37f944c9de88 2024-09-21T10:50:46.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:45 smithi175 ceph-mon[49304]: pgmap v372: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:50:47.063 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -e sha1=5c1cdc3c470dae78e91a0ed2bcf13509a046d301 -- bash -c 'ceph versions' 2024-09-21T10:50:48.350 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:47 smithi123 ceph-mon[63856]: pgmap v373: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T10:50:48.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:47 smithi123 ceph-mon[63856]: from='client.34592 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:50:48.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:47 smithi175 ceph-mon[49304]: pgmap v373: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T10:50:48.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:47 smithi175 ceph-mon[49304]: from='client.34592 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:50:49.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:48 smithi123 ceph-mon[63856]: pgmap v374: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:50:49.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:48 smithi175 ceph-mon[49304]: pgmap v374: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:50:49.395 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-09-21T10:50:49.395 INFO:teuthology.orchestra.run.smithi123.stdout: "mon": { 2024-09-21T10:50:49.395 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev)": 2 2024-09-21T10:50:49.395 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:50:49.395 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr": { 2024-09-21T10:50:49.395 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev)": 2 2024-09-21T10:50:49.395 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:50:49.395 INFO:teuthology.orchestra.run.smithi123.stdout: "osd": { 2024-09-21T10:50:49.395 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev)": 8 2024-09-21T10:50:49.395 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:50:49.395 INFO:teuthology.orchestra.run.smithi123.stdout: "mds": { 2024-09-21T10:50:49.396 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev)": 2 2024-09-21T10:50:49.396 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:50:49.396 INFO:teuthology.orchestra.run.smithi123.stdout: "overall": { 2024-09-21T10:50:49.396 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev)": 14 2024-09-21T10:50:49.396 INFO:teuthology.orchestra.run.smithi123.stdout: } 2024-09-21T10:50:49.396 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-09-21T10:50:50.148 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:49 smithi123 ceph-mon[63856]: from='client.? 172.21.15.123:0/2578060643' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:50:50.170 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -e sha1=5c1cdc3c470dae78e91a0ed2bcf13509a046d301 -- bash -c 'echo "wait for servicemap items w/ changing names to refresh"' 2024-09-21T10:50:50.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:49 smithi175 ceph-mon[49304]: from='client.? 172.21.15.123:0/2578060643' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:50:51.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:50 smithi123 ceph-mon[63856]: pgmap v375: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:50:51.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:50 smithi175 ceph-mon[49304]: pgmap v375: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:50:51.950 INFO:teuthology.orchestra.run.smithi123.stdout:wait for servicemap items w/ changing names to refresh 2024-09-21T10:50:52.730 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -e sha1=5c1cdc3c470dae78e91a0ed2bcf13509a046d301 -- bash -c 'sleep 60' 2024-09-21T10:50:53.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:53 smithi123 ceph-mon[63856]: pgmap v376: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:50:53.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:53 smithi175 ceph-mon[49304]: pgmap v376: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:50:55.850 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:55 smithi123 ceph-mon[63856]: pgmap v377: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:50:55.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:55 smithi175 ceph-mon[49304]: pgmap v377: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:50:57.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:57 smithi123 ceph-mon[63856]: pgmap v378: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T10:50:57.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:57 smithi175 ceph-mon[49304]: pgmap v378: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T10:50:58.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:58 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:50:58.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:58 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:50:59.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:50:59 smithi123 ceph-mon[63856]: pgmap v379: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:50:59.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:50:59 smithi175 ceph-mon[49304]: pgmap v379: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:51:01.351 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:51:01 smithi123 ceph-mon[63856]: pgmap v380: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:51:01.373 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:51:01 smithi175 ceph-mon[49304]: pgmap v380: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:51:03.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:51:03 smithi123 ceph-mon[63856]: pgmap v381: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:51:03.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:51:03 smithi175 ceph-mon[49304]: pgmap v381: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:51:05.850 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:51:05 smithi123 ceph-mon[63856]: pgmap v382: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:51:05.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:51:05 smithi175 ceph-mon[49304]: pgmap v382: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:51:07.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:51:07 smithi123 ceph-mon[63856]: pgmap v383: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T10:51:07.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:51:07 smithi175 ceph-mon[49304]: pgmap v383: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T10:51:09.850 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:51:09 smithi123 ceph-mon[63856]: pgmap v384: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:51:09.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:51:09 smithi175 ceph-mon[49304]: pgmap v384: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:51:11.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:51:11 smithi123 ceph-mon[63856]: pgmap v385: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:51:11.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:51:11 smithi175 ceph-mon[49304]: pgmap v385: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:51:13.675 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:51:13 smithi175 ceph-mon[49304]: pgmap v386: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:51:13.675 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:51:13 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:51:13.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:51:13 smithi123 ceph-mon[63856]: pgmap v386: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:51:13.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:51:13 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:51:15.850 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:51:15 smithi123 ceph-mon[63856]: pgmap v387: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:51:15.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:51:15 smithi175 ceph-mon[49304]: pgmap v387: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:51:17.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:51:17 smithi123 ceph-mon[63856]: pgmap v388: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T10:51:17.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:51:17 smithi175 ceph-mon[49304]: pgmap v388: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T10:51:19.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:51:19 smithi123 ceph-mon[63856]: pgmap v389: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:51:19.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:51:19 smithi175 ceph-mon[49304]: pgmap v389: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:51:21.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:51:21 smithi123 ceph-mon[63856]: pgmap v390: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:51:21.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:51:21 smithi175 ceph-mon[49304]: pgmap v390: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:51:23.850 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:51:23 smithi123 ceph-mon[63856]: pgmap v391: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:51:23.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:51:23 smithi175 ceph-mon[49304]: pgmap v391: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:51:25.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:51:25 smithi123 ceph-mon[63856]: pgmap v392: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:51:25.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:51:25 smithi175 ceph-mon[49304]: pgmap v392: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:51:27.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:51:27 smithi123 ceph-mon[63856]: pgmap v393: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T10:51:27.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:51:27 smithi175 ceph-mon[49304]: pgmap v393: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T10:51:28.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:51:28 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:51:28.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:51:28 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:51:29.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:51:29 smithi123 ceph-mon[63856]: pgmap v394: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:51:29.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:51:29 smithi175 ceph-mon[49304]: pgmap v394: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:51:30.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:51:30 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:51:30.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:51:30 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:51:30.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:51:30 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:51:30.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:51:30 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:51:30.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:51:30 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:51:30.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:51:30 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:51:30.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:51:30 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:51:30.602 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:51:30 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:51:30.602 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:51:30 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:51:30.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:51:30 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:51:30.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:51:30 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:51:30.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:51:30 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:51:30.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:51:30 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:51:30.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:51:30 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:51:30.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:51:30 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:51:30.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:51:30 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:51:30.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:51:30 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:51:30.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:51:30 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' 2024-09-21T10:51:31.850 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:51:31 smithi123 ceph-mon[63856]: pgmap v395: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:51:31.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:51:31 smithi175 ceph-mon[49304]: pgmap v395: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:51:33.850 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:51:33 smithi123 ceph-mon[63856]: pgmap v396: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:51:33.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:51:33 smithi175 ceph-mon[49304]: pgmap v396: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:51:35.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:51:35 smithi123 ceph-mon[63856]: pgmap v397: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:51:35.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:51:35 smithi175 ceph-mon[49304]: pgmap v397: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:51:37.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:51:37 smithi123 ceph-mon[63856]: pgmap v398: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T10:51:37.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:51:37 smithi175 ceph-mon[49304]: pgmap v398: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T10:51:39.853 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:51:39 smithi123 ceph-mon[63856]: pgmap v399: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:51:39.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:51:39 smithi175 ceph-mon[49304]: pgmap v399: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:51:41.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:51:41 smithi123 ceph-mon[63856]: pgmap v400: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:51:41.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:51:41 smithi175 ceph-mon[49304]: pgmap v400: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:51:43.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:51:43 smithi123 ceph-mon[63856]: pgmap v401: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:51:43.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:51:43 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:51:43.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:51:43 smithi175 ceph-mon[49304]: pgmap v401: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:51:43.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:51:43 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:51:45.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:51:45 smithi123 ceph-mon[63856]: pgmap v402: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:51:45.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:51:45 smithi175 ceph-mon[49304]: pgmap v402: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:51:47.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:51:47 smithi123 ceph-mon[63856]: pgmap v403: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T10:51:47.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:51:47 smithi175 ceph-mon[49304]: pgmap v403: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T10:51:49.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:51:49 smithi123 ceph-mon[63856]: pgmap v404: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:51:49.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:51:49 smithi175 ceph-mon[49304]: pgmap v404: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:51:51.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:51:51 smithi123 ceph-mon[63856]: pgmap v405: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:51:51.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:51:51 smithi175 ceph-mon[49304]: pgmap v405: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:51:53.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:51:53 smithi123 ceph-mon[63856]: pgmap v406: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:51:53.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:51:53 smithi175 ceph-mon[49304]: pgmap v406: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:51:55.113 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -e sha1=5c1cdc3c470dae78e91a0ed2bcf13509a046d301 -- bash -c 'ceph orch ps' 2024-09-21T10:51:55.728 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:51:55 smithi123 ceph-mon[63856]: pgmap v407: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:51:55.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:51:55 smithi175 ceph-mon[49304]: pgmap v407: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:51:57.233 INFO:teuthology.orchestra.run.smithi123.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T10:51:57.233 INFO:teuthology.orchestra.run.smithi123.stdout:alertmanager.smithi123 smithi123 *:9093,9094 running (12m) 94s ago 21m 22.1M - 0.27.0 11f11916f8cd 5d550bd97de5 2024-09-21T10:51:57.233 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi123 smithi123 running (9m) 94s ago 21m 6899k - 19.3.0-5103-g5c1cdc3c 8f4d013db793 c5e1ef39a0a8 2024-09-21T10:51:57.234 INFO:teuthology.orchestra.run.smithi123.stdout:crash.smithi175 smithi175 running (9m) 2m ago 20m 6895k - 19.3.0-5103-g5c1cdc3c 8f4d013db793 fb1ca999ccfd 2024-09-21T10:51:57.234 INFO:teuthology.orchestra.run.smithi123.stdout:grafana.smithi123 smithi123 *:3000 running (100s) 94s ago 21m 36.9M - 10.4.8 5aad1d7cf004 af3591d01f0d 2024-09-21T10:51:57.234 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi123.uqoyow smithi123 running (2m) 94s ago 15m 30.7M - 19.3.0-5103-g5c1cdc3c 8f4d013db793 9b8c6a6dd421 2024-09-21T10:51:57.234 INFO:teuthology.orchestra.run.smithi123.stdout:mds.foofs.smithi175.jrlevw smithi175 running (2m) 2m ago 15m 13.2M - 19.3.0-5103-g5c1cdc3c 8f4d013db793 052f5731455a 2024-09-21T10:51:57.234 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi123.stnuit smithi123 *:8443,9283,8765 running (11m) 94s ago 22m 533M - 19.3.0-5103-g5c1cdc3c 8f4d013db793 6812cc5cb56b 2024-09-21T10:51:57.234 INFO:teuthology.orchestra.run.smithi123.stdout:mgr.smithi175.txvwrq smithi175 *:8443,9283,8765 running (10m) 2m ago 20m 457M - 19.3.0-5103-g5c1cdc3c 8f4d013db793 304f0408075e 2024-09-21T10:51:57.234 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi123 smithi123 running (10m) 94s ago 23m 65.6M 2048M 19.3.0-5103-g5c1cdc3c 8f4d013db793 6b95207a2d80 2024-09-21T10:51:57.234 INFO:teuthology.orchestra.run.smithi123.stdout:mon.smithi175 smithi175 running (9m) 2m ago 20m 50.0M 2048M 19.3.0-5103-g5c1cdc3c 8f4d013db793 3abee2d37f0f 2024-09-21T10:51:57.234 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.0.0.smithi123.ddatpr smithi123 *:2049 running (2m) 94s ago 15m 65.1M - 5.9 8f4d013db793 8d4f82293026 2024-09-21T10:51:57.234 INFO:teuthology.orchestra.run.smithi123.stdout:nfs.foo.1.0.smithi175.wpawer smithi175 *:2049 running (2m) 2m ago 15m 24.8M - 5.9 8f4d013db793 86d9bf42d82a 2024-09-21T10:51:57.234 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi123 smithi123 *:9100 running (12m) 94s ago 21m 12.2M - 1.7.0 72c9c2088986 ca7d0b29af81 2024-09-21T10:51:57.235 INFO:teuthology.orchestra.run.smithi123.stdout:node-exporter.smithi175 smithi175 *:9100 running (11m) 2m ago 20m 12.2M - 1.7.0 72c9c2088986 46c4f6b7c149 2024-09-21T10:51:57.235 INFO:teuthology.orchestra.run.smithi123.stdout:osd.0 smithi123 running (8m) 94s ago 20m 438M 1936M 19.3.0-5103-g5c1cdc3c 8f4d013db793 cc40f695c3a8 2024-09-21T10:51:57.235 INFO:teuthology.orchestra.run.smithi123.stdout:osd.1 smithi123 running (7m) 94s ago 19m 321M 1936M 19.3.0-5103-g5c1cdc3c 8f4d013db793 1be249025c7f 2024-09-21T10:51:57.235 INFO:teuthology.orchestra.run.smithi123.stdout:osd.2 smithi123 running (6m) 94s ago 19m 300M 1936M 19.3.0-5103-g5c1cdc3c 8f4d013db793 647aec759e3c 2024-09-21T10:51:57.235 INFO:teuthology.orchestra.run.smithi123.stdout:osd.3 smithi123 running (5m) 94s ago 18m 415M 1936M 19.3.0-5103-g5c1cdc3c 8f4d013db793 7a082fcad8e8 2024-09-21T10:51:57.235 INFO:teuthology.orchestra.run.smithi123.stdout:osd.4 smithi175 running (5m) 2m ago 18m 348M 2704M 19.3.0-5103-g5c1cdc3c 8f4d013db793 b650f72d10f0 2024-09-21T10:51:57.235 INFO:teuthology.orchestra.run.smithi123.stdout:osd.5 smithi175 running (4m) 2m ago 17m 338M 2704M 19.3.0-5103-g5c1cdc3c 8f4d013db793 8199a0b362df 2024-09-21T10:51:57.235 INFO:teuthology.orchestra.run.smithi123.stdout:osd.6 smithi175 running (3m) 2m ago 17m 313M 2704M 19.3.0-5103-g5c1cdc3c 8f4d013db793 aab90bc08918 2024-09-21T10:51:57.235 INFO:teuthology.orchestra.run.smithi123.stdout:osd.7 smithi175 running (3m) 2m ago 16m 304M 2704M 19.3.0-5103-g5c1cdc3c 8f4d013db793 2f37c8ff343f 2024-09-21T10:51:57.235 INFO:teuthology.orchestra.run.smithi123.stdout:prometheus.smithi123 smithi123 *:9095 running (10m) 94s ago 20m 58.1M - 2.51.0 1d3b7f56885b 37f944c9de88 2024-09-21T10:51:57.753 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:51:57 smithi123 ceph-mon[63856]: pgmap v408: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T10:51:57.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:51:57 smithi175 ceph-mon[49304]: pgmap v408: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T10:51:57.924 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -e sha1=5c1cdc3c470dae78e91a0ed2bcf13509a046d301 -- bash -c 'ceph orch upgrade status' 2024-09-21T10:51:58.850 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:51:58 smithi123 ceph-mon[63856]: from='client.34600 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:51:58.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:51:58 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:51:58.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:51:58 smithi175 ceph-mon[49304]: from='client.34600 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:51:58.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:51:58 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:51:59.850 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:51:59 smithi123 ceph-mon[63856]: pgmap v409: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:51:59.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:51:59 smithi175 ceph-mon[49304]: pgmap v409: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:51:59.945 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-09-21T10:51:59.945 INFO:teuthology.orchestra.run.smithi123.stdout: "target_image": null, 2024-09-21T10:51:59.945 INFO:teuthology.orchestra.run.smithi123.stdout: "in_progress": false, 2024-09-21T10:51:59.945 INFO:teuthology.orchestra.run.smithi123.stdout: "which": "", 2024-09-21T10:51:59.945 INFO:teuthology.orchestra.run.smithi123.stdout: "services_complete": [], 2024-09-21T10:51:59.945 INFO:teuthology.orchestra.run.smithi123.stdout: "progress": null, 2024-09-21T10:51:59.945 INFO:teuthology.orchestra.run.smithi123.stdout: "message": "", 2024-09-21T10:51:59.945 INFO:teuthology.orchestra.run.smithi123.stdout: "is_paused": false 2024-09-21T10:51:59.945 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-09-21T10:52:00.653 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -e sha1=5c1cdc3c470dae78e91a0ed2bcf13509a046d301 -- bash -c 'ceph health detail' 2024-09-21T10:52:01.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:52:01 smithi123 ceph-mon[63856]: from='client.34604 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:52:01.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:52:01 smithi123 ceph-mon[63856]: pgmap v410: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:52:01.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:52:01 smithi175 ceph-mon[49304]: from='client.34604 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:52:01.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:52:01 smithi175 ceph-mon[49304]: pgmap v410: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:52:02.620 INFO:teuthology.orchestra.run.smithi123.stdout:HEALTH_OK 2024-09-21T10:52:03.208 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -e sha1=5c1cdc3c470dae78e91a0ed2bcf13509a046d301 -- bash -c 'ceph versions' 2024-09-21T10:52:03.829 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:52:03 smithi123 ceph-mon[63856]: pgmap v411: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:52:03.829 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:52:03 smithi123 ceph-mon[63856]: from='client.? 172.21.15.123:0/1974052411' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T10:52:03.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:52:03 smithi175 ceph-mon[49304]: pgmap v411: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:52:03.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:52:03 smithi175 ceph-mon[49304]: from='client.? 172.21.15.123:0/1974052411' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T10:52:05.305 INFO:teuthology.orchestra.run.smithi123.stdout:{ 2024-09-21T10:52:05.305 INFO:teuthology.orchestra.run.smithi123.stdout: "mon": { 2024-09-21T10:52:05.306 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev)": 2 2024-09-21T10:52:05.306 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:52:05.306 INFO:teuthology.orchestra.run.smithi123.stdout: "mgr": { 2024-09-21T10:52:05.306 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev)": 2 2024-09-21T10:52:05.306 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:52:05.306 INFO:teuthology.orchestra.run.smithi123.stdout: "osd": { 2024-09-21T10:52:05.306 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev)": 8 2024-09-21T10:52:05.306 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:52:05.306 INFO:teuthology.orchestra.run.smithi123.stdout: "mds": { 2024-09-21T10:52:05.306 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev)": 2 2024-09-21T10:52:05.306 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:52:05.306 INFO:teuthology.orchestra.run.smithi123.stdout: "overall": { 2024-09-21T10:52:05.307 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev)": 14 2024-09-21T10:52:05.307 INFO:teuthology.orchestra.run.smithi123.stdout: } 2024-09-21T10:52:05.307 INFO:teuthology.orchestra.run.smithi123.stdout:} 2024-09-21T10:52:05.600 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:52:05 smithi123 ceph-mon[63856]: pgmap v412: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:52:05.600 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:52:05 smithi123 ceph-mon[63856]: from='client.? 172.21.15.123:0/1972842451' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:52:05.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:52:05 smithi175 ceph-mon[49304]: pgmap v412: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:52:05.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:52:05 smithi175 ceph-mon[49304]: from='client.? 172.21.15.123:0/1972842451' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:52:06.056 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -e sha1=5c1cdc3c470dae78e91a0ed2bcf13509a046d301 -- bash -c 'ceph versions | jq -e '"'"'.overall | length == 1'"'"'' 2024-09-21T10:52:07.600 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:52:07 smithi123 ceph-mon[63856]: pgmap v413: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T10:52:07.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:52:07 smithi175 ceph-mon[49304]: pgmap v413: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T10:52:08.185 INFO:teuthology.orchestra.run.smithi123.stdout:true 2024-09-21T10:52:08.846 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:52:08 smithi123 ceph-mon[63856]: from='client.? 172.21.15.123:0/1247725763' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:52:08.872 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -e sha1=5c1cdc3c470dae78e91a0ed2bcf13509a046d301 -- bash -c 'ceph versions | jq -e '"'"'.overall | keys'"'"' | grep $sha1' 2024-09-21T10:52:08.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:52:08 smithi175 ceph-mon[49304]: from='client.? 172.21.15.123:0/1247725763' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:52:09.837 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:52:09 smithi123 ceph-mon[63856]: pgmap v414: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:52:09.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:52:09 smithi175 ceph-mon[49304]: pgmap v414: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:52:10.935 INFO:teuthology.orchestra.run.smithi123.stdout: "ceph version 19.3.0-5103-g5c1cdc3c (5c1cdc3c470dae78e91a0ed2bcf13509a046d301) squid (dev)" 2024-09-21T10:52:11.551 INFO:teuthology.task.sequential:In sequential, running task cephadm.wait_for_service... 2024-09-21T10:52:11.551 INFO:tasks.cephadm:Waiting for ceph service nfs.foo to start (timeout 300)... 2024-09-21T10:52:11.552 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -- ceph orch ls -f json 2024-09-21T10:52:11.847 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:52:11 smithi123 ceph-mon[63856]: pgmap v415: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:52:11.847 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:52:11 smithi123 ceph-mon[63856]: from='client.? 172.21.15.123:0/4044989947' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:52:11.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:52:11 smithi175 ceph-mon[49304]: pgmap v415: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:52:11.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:52:11 smithi175 ceph-mon[49304]: from='client.? 172.21.15.123:0/4044989947' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:52:13.582 INFO:teuthology.orchestra.run.smithi123.stdout: 2024-09-21T10:52:13.582 INFO:teuthology.orchestra.run.smithi123.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-21T10:29:47.207278Z", "last_refresh": "2024-09-21T10:50:22.353674Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-21T10:29:40.698729Z", "last_refresh": "2024-09-21T10:49:55.278926Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "spec": {"anonymous_access": true, "protocol": "https"}, "status": {"created": "2024-09-21T10:29:44.033793Z", "last_refresh": "2024-09-21T10:50:22.353813Z", "ports": [3000], "running": 1, "size": 1}}, {"placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-21T10:36:25.784744Z", "last_refresh": "2024-09-21T10:49:55.279488Z", "running": 2, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-21T10:29:38.874687Z", "last_refresh": "2024-09-21T10:49:55.279020Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi123:172.21.15.123=smithi123", "smithi175:172.21.15.175=smithi175"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-21T10:30:30.797624Z", "last_refresh": "2024-09-21T10:49:55.279091Z", "running": 2, "size": 2}}, {"events": ["2024-09-21T10:51:30.301038Z service:nfs.foo [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foo", "service_name": "nfs.foo", "service_type": "nfs", "spec": {"port": 2049}, "status": {"created": "2024-09-21T10:36:42.887205Z", "last_refresh": "2024-09-21T10:49:55.279551Z", "ports": [2049], "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-21T10:29:45.863421Z", "last_refresh": "2024-09-21T10:49:55.279161Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "8f4d013db793795fef09ff5edf96e6b42ba8cd2c9158b2f0b7a5d7c1e657c063", "container_image_name": "quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472", "last_refresh": "2024-09-21T10:49:55.279241Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-21T10:29:42.699451Z", "last_refresh": "2024-09-21T10:50:22.353959Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-21T10:52:13.850 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:52:13 smithi123 ceph-mon[63856]: pgmap v416: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:52:13.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:52:13 smithi123 ceph-mon[63856]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:52:13.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:52:13 smithi175 ceph-mon[49304]: pgmap v416: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:52:13.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:52:13 smithi175 ceph-mon[49304]: from='mgr.34102 172.21.15.123:0/4210836590' entity='mgr.smithi123.stnuit' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:52:14.197 INFO:tasks.cephadm:nfs.foo has 2/2 2024-09-21T10:52:14.197 DEBUG:teuthology.parallel:result is None 2024-09-21T10:52:14.198 INFO:teuthology.run_tasks:Running task vip.exec... 2024-09-21T10:52:14.207 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi123.front.sepia.ceph.com 2024-09-21T10:52:14.207 DEBUG:teuthology.orchestra.run.smithi123:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'umount /mnt/foo' 2024-09-21T10:52:14.233 INFO:teuthology.orchestra.run.smithi123.stderr:+ umount /mnt/foo 2024-09-21T10:52:14.258 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-21T10:52:14.268 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi123.front.sepia.ceph.com 2024-09-21T10:52:14.268 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -- bash -c 'ceph nfs cluster ls | grep foo' 2024-09-21T10:52:14.844 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:52:14 smithi123 ceph-mon[63856]: from='client.34624 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:52:14.874 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:52:14 smithi175 ceph-mon[49304]: from='client.34624 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:52:15.599 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:52:15 smithi123 ceph-mon[63856]: pgmap v417: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:52:15.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:52:15 smithi175 ceph-mon[49304]: pgmap v417: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:52:16.474 INFO:teuthology.orchestra.run.smithi123.stdout: "foo" 2024-09-21T10:52:17.046 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -- bash -c 'ceph nfs export ls foo --detailed' 2024-09-21T10:52:17.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:52:17 smithi123 ceph-mon[63856]: pgmap v418: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T10:52:17.601 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:52:17 smithi123 ceph-mon[63856]: from='client.34628 -' entity='client.admin' cmd=[{"prefix": "nfs cluster ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:52:17.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:52:17 smithi175 ceph-mon[49304]: pgmap v418: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T10:52:17.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:52:17 smithi175 ceph-mon[49304]: from='client.34628 -' entity='client.admin' cmd=[{"prefix": "nfs cluster ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:52:19.203 INFO:teuthology.orchestra.run.smithi123.stdout:[ 2024-09-21T10:52:19.203 INFO:teuthology.orchestra.run.smithi123.stdout: { 2024-09-21T10:52:19.203 INFO:teuthology.orchestra.run.smithi123.stdout: "access_type": "RW", 2024-09-21T10:52:19.204 INFO:teuthology.orchestra.run.smithi123.stdout: "clients": [], 2024-09-21T10:52:19.204 INFO:teuthology.orchestra.run.smithi123.stdout: "cluster_id": "foo", 2024-09-21T10:52:19.204 INFO:teuthology.orchestra.run.smithi123.stdout: "export_id": 1, 2024-09-21T10:52:19.204 INFO:teuthology.orchestra.run.smithi123.stdout: "fsal": { 2024-09-21T10:52:19.204 INFO:teuthology.orchestra.run.smithi123.stdout: "fs_name": "foofs", 2024-09-21T10:52:19.204 INFO:teuthology.orchestra.run.smithi123.stdout: "name": "CEPH", 2024-09-21T10:52:19.204 INFO:teuthology.orchestra.run.smithi123.stdout: "user_id": "nfs.foo.1" 2024-09-21T10:52:19.204 INFO:teuthology.orchestra.run.smithi123.stdout: }, 2024-09-21T10:52:19.204 INFO:teuthology.orchestra.run.smithi123.stdout: "path": "/", 2024-09-21T10:52:19.204 INFO:teuthology.orchestra.run.smithi123.stdout: "protocols": [ 2024-09-21T10:52:19.204 INFO:teuthology.orchestra.run.smithi123.stdout: 4 2024-09-21T10:52:19.204 INFO:teuthology.orchestra.run.smithi123.stdout: ], 2024-09-21T10:52:19.205 INFO:teuthology.orchestra.run.smithi123.stdout: "pseudo": "/fake", 2024-09-21T10:52:19.205 INFO:teuthology.orchestra.run.smithi123.stdout: "security_label": true, 2024-09-21T10:52:19.205 INFO:teuthology.orchestra.run.smithi123.stdout: "squash": "none", 2024-09-21T10:52:19.205 INFO:teuthology.orchestra.run.smithi123.stdout: "transports": [ 2024-09-21T10:52:19.205 INFO:teuthology.orchestra.run.smithi123.stdout: "TCP" 2024-09-21T10:52:19.205 INFO:teuthology.orchestra.run.smithi123.stdout: ] 2024-09-21T10:52:19.205 INFO:teuthology.orchestra.run.smithi123.stdout: } 2024-09-21T10:52:19.205 INFO:teuthology.orchestra.run.smithi123.stdout:] 2024-09-21T10:52:19.850 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:52:19 smithi123 ceph-mon[63856]: pgmap v419: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:52:19.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:52:19 smithi175 ceph-mon[49304]: pgmap v419: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:52:19.910 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -- bash -c 'rados -p .nfs --all ls -' 2024-09-21T10:52:20.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:52:20 smithi123 ceph-mon[63856]: from='client.34632 -' entity='client.admin' cmd=[{"prefix": "nfs export ls", "cluster_id": "foo", "detailed": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:52:20.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:52:20 smithi175 ceph-mon[49304]: from='client.34632 -' entity='client.admin' cmd=[{"prefix": "nfs export ls", "cluster_id": "foo", "detailed": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:52:21.730 INFO:teuthology.orchestra.run.smithi123.stdout:foo rec-0000000000000004:nfs.foo.1 2024-09-21T10:52:21.730 INFO:teuthology.orchestra.run.smithi123.stdout:foo grace 2024-09-21T10:52:21.731 INFO:teuthology.orchestra.run.smithi123.stdout:foo rec-0000000000000004:nfs.foo.0 2024-09-21T10:52:21.731 INFO:teuthology.orchestra.run.smithi123.stdout:foo conf-nfs.foo 2024-09-21T10:52:21.731 INFO:teuthology.orchestra.run.smithi123.stdout:foo export-1 2024-09-21T10:52:21.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:52:21 smithi123 ceph-mon[63856]: pgmap v420: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:52:21.851 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:52:21 smithi123 ceph-mon[63856]: mgrmap e42: smithi123.stnuit(active, since 10m), standbys: smithi175.txvwrq 2024-09-21T10:52:21.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:52:21 smithi175 ceph-mon[49304]: pgmap v420: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:52:21.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:52:21 smithi175 ceph-mon[49304]: mgrmap e42: smithi123.stnuit(active, since 10m), standbys: smithi175.txvwrq 2024-09-21T10:52:22.349 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-21T10:52:22.359 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi123.front.sepia.ceph.com 2024-09-21T10:52:22.359 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 -- bash -c 'set -ex 2024-09-21T10:52:22.359 DEBUG:teuthology.orchestra.run.smithi123:> [[ `ceph config get mgr mgr/cephadm/migration_current` -gt 2 ]] 2024-09-21T10:52:22.359 DEBUG:teuthology.orchestra.run.smithi123:> ' 2024-09-21T10:52:23.667 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:52:23 smithi123 ceph-mon[63856]: pgmap v421: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:52:23.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:52:23 smithi175 ceph-mon[49304]: pgmap v421: 97 pgs: 97 active+clean; 230 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T10:52:24.144 INFO:teuthology.orchestra.run.smithi123.stderr:++ ceph config get mgr mgr/cephadm/migration_current 2024-09-21T10:52:24.405 INFO:teuthology.orchestra.run.smithi123.stderr:+ [[ 7 -gt 2 ]] 2024-09-21T10:52:24.689 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:52:24 smithi123 ceph-mon[63856]: from='client.? 172.21.15.123:0/2356662845' entity='client.admin' cmd={"prefix": "config get", "who": "mgr", "key": "mgr/cephadm/migration_current"} : dispatch 2024-09-21T10:52:24.873 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:52:24 smithi175 ceph-mon[49304]: from='client.? 172.21.15.123:0/2356662845' entity='client.admin' cmd={"prefix": "config get", "who": "mgr", "key": "mgr/cephadm/migration_current"} : dispatch 2024-09-21T10:52:25.005 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-09-21T10:52:25.020 INFO:tasks.cephadm:Teardown begin 2024-09-21T10:52:25.021 DEBUG:teuthology.orchestra.run.smithi123:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-21T10:52:25.048 DEBUG:teuthology.orchestra.run.smithi175:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-21T10:52:25.082 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-09-21T10:52:25.082 DEBUG:teuthology.orchestra.run.smithi123:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-09-21T10:52:25.103 DEBUG:teuthology.orchestra.run.smithi175:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-09-21T10:52:25.140 INFO:tasks.cephadm:Stopping all daemons... 2024-09-21T10:52:25.141 INFO:tasks.cephadm.mon.smithi123:Stopping mon.smithi123... 2024-09-21T10:52:25.141 DEBUG:teuthology.orchestra.run.smithi123:> sudo systemctl stop ceph-3ccd15e6-7804-11ef-baf6-efdc52797490@mon.smithi123 2024-09-21T10:52:25.350 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:52:25 smithi123 systemd[1]: Stopping Ceph mon.smithi123 for 3ccd15e6-7804-11ef-baf6-efdc52797490... 2024-09-21T10:52:25.742 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:52:25 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-mon-smithi123[63852]: 2024-09-21T10:52:25.442+0000 7f608bf74640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi123 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:52:25.742 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:52:25 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-mon-smithi123[63852]: 2024-09-21T10:52:25.442+0000 7f608bf74640 -1 mon.smithi123@0(leader) e3 *** Got Signal Terminated *** 2024-09-21T10:52:25.742 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:52:25 smithi123 podman[93782]: 2024-09-21 10:52:25.493584371 +0000 UTC m=+0.171357269 container died 6b95207a2d807be3959310629d20a58fbce105e32dffa46adcf09109666a4896 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-mon-smithi123, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T10:52:26.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:52:25 smithi123 podman[93782]: 2024-09-21 10:52:25.858255123 +0000 UTC m=+0.536028028 container cleanup 6b95207a2d807be3959310629d20a58fbce105e32dffa46adcf09109666a4896 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-mon-smithi123, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2) 2024-09-21T10:52:26.101 INFO:journalctl@ceph.mon.smithi123.smithi123.stdout:Sep 21 10:52:25 smithi123 bash[93782]: ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-mon-smithi123 2024-09-21T10:52:26.285 DEBUG:teuthology.orchestra.run.smithi123:> sudo pkill -f 'journalctl -f -n 0 -u ceph-3ccd15e6-7804-11ef-baf6-efdc52797490@mon.smithi123.service' 2024-09-21T10:52:26.330 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T10:52:26.330 INFO:tasks.cephadm.mon.smithi123:Stopped mon.smithi123 2024-09-21T10:52:26.331 INFO:tasks.cephadm.mon.smithi175:Stopping mon.smithi175... 2024-09-21T10:52:26.331 DEBUG:teuthology.orchestra.run.smithi175:> sudo systemctl stop ceph-3ccd15e6-7804-11ef-baf6-efdc52797490@mon.smithi175 2024-09-21T10:52:26.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:52:26 smithi175 systemd[1]: Stopping Ceph mon.smithi175 for 3ccd15e6-7804-11ef-baf6-efdc52797490... 2024-09-21T10:52:27.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:52:26 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-mon-smithi175[49300]: 2024-09-21T10:52:26.736+0000 7f9f46a4f640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi175 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:52:27.123 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:52:26 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-mon-smithi175[49300]: 2024-09-21T10:52:26.736+0000 7f9f46a4f640 -1 mon.smithi175@1(peon) e3 *** Got Signal Terminated *** 2024-09-21T10:52:27.124 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:52:26 smithi175 podman[68406]: 2024-09-21 10:52:26.941154768 +0000 UTC m=+0.417671893 container died 3abee2d37f0f1c7e54993398d30da21d183d47b7a44609532666b16d2f74b204 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-mon-smithi175, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T10:52:27.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:52:27 smithi175 podman[68406]: 2024-09-21 10:52:27.428742712 +0000 UTC m=+0.905259836 container cleanup 3abee2d37f0f1c7e54993398d30da21d183d47b7a44609532666b16d2f74b204 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-mon-smithi175, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731) 2024-09-21T10:52:27.623 INFO:journalctl@ceph.mon.smithi175.smithi175.stdout:Sep 21 10:52:27 smithi175 bash[68406]: ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-mon-smithi175 2024-09-21T10:52:27.856 DEBUG:teuthology.orchestra.run.smithi175:> sudo pkill -f 'journalctl -f -n 0 -u ceph-3ccd15e6-7804-11ef-baf6-efdc52797490@mon.smithi175.service' 2024-09-21T10:52:27.902 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T10:52:27.902 INFO:tasks.cephadm.mon.smithi175:Stopped mon.smithi175 2024-09-21T10:52:27.903 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-09-21T10:52:27.903 DEBUG:teuthology.orchestra.run.smithi123:> sudo systemctl stop ceph-3ccd15e6-7804-11ef-baf6-efdc52797490@osd.0 2024-09-21T10:52:28.206 INFO:journalctl@ceph.osd.0.smithi123.stdout:Sep 21 10:52:27 smithi123 systemd[1]: Stopping Ceph osd.0 for 3ccd15e6-7804-11ef-baf6-efdc52797490... 2024-09-21T10:52:28.601 INFO:journalctl@ceph.osd.0.smithi123.stdout:Sep 21 10:52:28 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-0[68680]: 2024-09-21T10:52:28.203+0000 7ff558dbb640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:52:28.601 INFO:journalctl@ceph.osd.0.smithi123.stdout:Sep 21 10:52:28 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-0[68680]: 2024-09-21T10:52:28.203+0000 7ff558dbb640 -1 osd.0 174 *** Got signal Terminated *** 2024-09-21T10:52:28.601 INFO:journalctl@ceph.osd.0.smithi123.stdout:Sep 21 10:52:28 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-0[68680]: 2024-09-21T10:52:28.203+0000 7ff558dbb640 -1 osd.0 174 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T10:52:33.948 INFO:journalctl@ceph.osd.0.smithi123.stdout:Sep 21 10:52:33 smithi123 podman[93913]: 2024-09-21 10:52:33.683545903 +0000 UTC m=+5.587607593 container died cc40f695c3a88cc6d6448d93b429fc2f60765828ab0657e4d7598ee26bb1d5dd (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, io.buildah.version=1.37.2, GIT_CLEAN=True) 2024-09-21T10:52:34.351 INFO:journalctl@ceph.osd.0.smithi123.stdout:Sep 21 10:52:34 smithi123 podman[93913]: 2024-09-21 10:52:34.029290544 +0000 UTC m=+5.933352223 container cleanup cc40f695c3a88cc6d6448d93b429fc2f60765828ab0657e4d7598ee26bb1d5dd (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-0, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-09-21T10:52:34.351 INFO:journalctl@ceph.osd.0.smithi123.stdout:Sep 21 10:52:34 smithi123 bash[93913]: ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-0 2024-09-21T10:52:34.676 INFO:journalctl@ceph.osd.0.smithi123.stdout:Sep 21 10:52:34 smithi123 podman[93947]: 2024-09-21 10:52:34.355499304 +0000 UTC m=+0.675827558 container remove cc40f695c3a88cc6d6448d93b429fc2f60765828ab0657e4d7598ee26bb1d5dd (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, org.label-schema.license=GPLv2) 2024-09-21T10:52:34.944 INFO:journalctl@ceph.osd.0.smithi123.stdout:Sep 21 10:52:34 smithi123 podman[94032]: 2024-09-21 10:52:34.578154254 +0000 UTC m=+0.016375388 image pull 8f4d013db793795fef09ff5edf96e6b42ba8cd2c9158b2f0b7a5d7c1e657c063 quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472 2024-09-21T10:52:34.944 INFO:journalctl@ceph.osd.0.smithi123.stdout:Sep 21 10:52:34 smithi123 podman[94032]: 2024-09-21 10:52:34.706612718 +0000 UTC m=+0.144833838 container create 7775bc67faf74db6c1566ea9aeaf814a7c604f113ea82407d67091ec12711dcb (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-0-deactivate, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731) 2024-09-21T10:52:34.944 INFO:journalctl@ceph.osd.0.smithi123.stdout:Sep 21 10:52:34 smithi123 podman[94032]: 2024-09-21 10:52:34.942391373 +0000 UTC m=+0.380612506 container init 7775bc67faf74db6c1566ea9aeaf814a7c604f113ea82407d67091ec12711dcb (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-0-deactivate, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T10:52:35.350 INFO:journalctl@ceph.osd.0.smithi123.stdout:Sep 21 10:52:34 smithi123 podman[94032]: 2024-09-21 10:52:34.946160777 +0000 UTC m=+0.384381910 container start 7775bc67faf74db6c1566ea9aeaf814a7c604f113ea82407d67091ec12711dcb (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-0-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-09-21T10:52:35.351 INFO:journalctl@ceph.osd.0.smithi123.stdout:Sep 21 10:52:35 smithi123 podman[94032]: 2024-09-21 10:52:35.017321683 +0000 UTC m=+0.455542818 container attach 7775bc67faf74db6c1566ea9aeaf814a7c604f113ea82407d67091ec12711dcb (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-0-deactivate, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T10:52:35.351 INFO:journalctl@ceph.osd.0.smithi123.stdout:Sep 21 10:52:35 smithi123 podman[94032]: 2024-09-21 10:52:35.135999224 +0000 UTC m=+0.574220348 container died 7775bc67faf74db6c1566ea9aeaf814a7c604f113ea82407d67091ec12711dcb (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-0-deactivate, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, ceph=True, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3) 2024-09-21T10:52:35.751 DEBUG:teuthology.orchestra.run.smithi123:> sudo pkill -f 'journalctl -f -n 0 -u ceph-3ccd15e6-7804-11ef-baf6-efdc52797490@osd.0.service' 2024-09-21T10:52:35.791 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T10:52:35.792 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-09-21T10:52:35.792 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-09-21T10:52:35.792 DEBUG:teuthology.orchestra.run.smithi123:> sudo systemctl stop ceph-3ccd15e6-7804-11ef-baf6-efdc52797490@osd.1 2024-09-21T10:52:36.101 INFO:journalctl@ceph.osd.1.smithi123.stdout:Sep 21 10:52:35 smithi123 systemd[1]: Stopping Ceph osd.1 for 3ccd15e6-7804-11ef-baf6-efdc52797490... 2024-09-21T10:52:36.600 INFO:journalctl@ceph.osd.1.smithi123.stdout:Sep 21 10:52:36 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-1[71214]: 2024-09-21T10:52:36.148+0000 7f419e627640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:52:36.601 INFO:journalctl@ceph.osd.1.smithi123.stdout:Sep 21 10:52:36 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-1[71214]: 2024-09-21T10:52:36.148+0000 7f419e627640 -1 osd.1 174 *** Got signal Terminated *** 2024-09-21T10:52:36.601 INFO:journalctl@ceph.osd.1.smithi123.stdout:Sep 21 10:52:36 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-1[71214]: 2024-09-21T10:52:36.148+0000 7f419e627640 -1 osd.1 174 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T10:52:41.902 INFO:journalctl@ceph.osd.1.smithi123.stdout:Sep 21 10:52:41 smithi123 podman[94134]: 2024-09-21 10:52:41.627402159 +0000 UTC m=+5.604128640 container died 1be249025c7f0b374c5328a4e25918f320c228f97c09cb5c6026bdb56cdedb07 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-1, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3) 2024-09-21T10:52:42.296 INFO:journalctl@ceph.osd.1.smithi123.stdout:Sep 21 10:52:41 smithi123 podman[94134]: 2024-09-21 10:52:41.999126552 +0000 UTC m=+5.975853030 container cleanup 1be249025c7f0b374c5328a4e25918f320c228f97c09cb5c6026bdb56cdedb07 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-1, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T10:52:42.296 INFO:journalctl@ceph.osd.1.smithi123.stdout:Sep 21 10:52:41 smithi123 bash[94134]: ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-1 2024-09-21T10:52:42.601 INFO:journalctl@ceph.osd.1.smithi123.stdout:Sep 21 10:52:42 smithi123 podman[94149]: 2024-09-21 10:52:42.294784371 +0000 UTC m=+0.666167932 container remove 1be249025c7f0b374c5328a4e25918f320c228f97c09cb5c6026bdb56cdedb07 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-1, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, ceph=True, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T10:52:42.980 INFO:journalctl@ceph.osd.1.smithi123.stdout:Sep 21 10:52:42 smithi123 podman[94240]: 2024-09-21 10:52:42.564699349 +0000 UTC m=+0.016109650 image pull 8f4d013db793795fef09ff5edf96e6b42ba8cd2c9158b2f0b7a5d7c1e657c063 quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472 2024-09-21T10:52:42.980 INFO:journalctl@ceph.osd.1.smithi123.stdout:Sep 21 10:52:42 smithi123 podman[94240]: 2024-09-21 10:52:42.667708043 +0000 UTC m=+0.119118345 container create 581fc4e894de388798bf5ce46d3f66938abec04336a886ac34309b369847bfd0 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-1-deactivate, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T10:52:42.980 INFO:journalctl@ceph.osd.1.smithi123.stdout:Sep 21 10:52:42 smithi123 podman[94240]: 2024-09-21 10:52:42.895438422 +0000 UTC m=+0.346848729 container init 581fc4e894de388798bf5ce46d3f66938abec04336a886ac34309b369847bfd0 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-1-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-09-21T10:52:42.980 INFO:journalctl@ceph.osd.1.smithi123.stdout:Sep 21 10:52:42 smithi123 podman[94240]: 2024-09-21 10:52:42.899020717 +0000 UTC m=+0.350431020 container start 581fc4e894de388798bf5ce46d3f66938abec04336a886ac34309b369847bfd0 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-1-deactivate, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-09-21T10:52:42.980 INFO:journalctl@ceph.osd.1.smithi123.stdout:Sep 21 10:52:42 smithi123 podman[94240]: 2024-09-21 10:52:42.97869111 +0000 UTC m=+0.430101422 container attach 581fc4e894de388798bf5ce46d3f66938abec04336a886ac34309b369847bfd0 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-1-deactivate, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, org.label-schema.vendor=CentOS) 2024-09-21T10:52:43.351 INFO:journalctl@ceph.osd.1.smithi123.stdout:Sep 21 10:52:43 smithi123 podman[94240]: 2024-09-21 10:52:43.090881696 +0000 UTC m=+0.542291999 container died 581fc4e894de388798bf5ce46d3f66938abec04336a886ac34309b369847bfd0 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-1-deactivate, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-09-21T10:52:43.661 DEBUG:teuthology.orchestra.run.smithi123:> sudo pkill -f 'journalctl -f -n 0 -u ceph-3ccd15e6-7804-11ef-baf6-efdc52797490@osd.1.service' 2024-09-21T10:52:43.701 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T10:52:43.701 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-09-21T10:52:43.701 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-09-21T10:52:43.702 DEBUG:teuthology.orchestra.run.smithi123:> sudo systemctl stop ceph-3ccd15e6-7804-11ef-baf6-efdc52797490@osd.2 2024-09-21T10:52:44.037 INFO:journalctl@ceph.osd.2.smithi123.stdout:Sep 21 10:52:43 smithi123 systemd[1]: Stopping Ceph osd.2 for 3ccd15e6-7804-11ef-baf6-efdc52797490... 2024-09-21T10:52:44.351 INFO:journalctl@ceph.osd.2.smithi123.stdout:Sep 21 10:52:44 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-2[73624]: 2024-09-21T10:52:44.034+0000 7fb591e58640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:52:44.351 INFO:journalctl@ceph.osd.2.smithi123.stdout:Sep 21 10:52:44 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-2[73624]: 2024-09-21T10:52:44.034+0000 7fb591e58640 -1 osd.2 174 *** Got signal Terminated *** 2024-09-21T10:52:44.351 INFO:journalctl@ceph.osd.2.smithi123.stdout:Sep 21 10:52:44 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-2[73624]: 2024-09-21T10:52:44.034+0000 7fb591e58640 -1 osd.2 174 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T10:52:49.730 INFO:journalctl@ceph.osd.2.smithi123.stdout:Sep 21 10:52:49 smithi123 podman[94343]: 2024-09-21 10:52:49.446850488 +0000 UTC m=+5.517060293 container died 647aec759e3c2f1473b2fea786ab91eee17aff7e2f503e41018d8b226c11ffd2 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-2, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T10:52:50.079 INFO:journalctl@ceph.osd.2.smithi123.stdout:Sep 21 10:52:49 smithi123 podman[94343]: 2024-09-21 10:52:49.809940184 +0000 UTC m=+5.880149977 container cleanup 647aec759e3c2f1473b2fea786ab91eee17aff7e2f503e41018d8b226c11ffd2 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-2, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731) 2024-09-21T10:52:50.079 INFO:journalctl@ceph.osd.2.smithi123.stdout:Sep 21 10:52:49 smithi123 bash[94343]: ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-2 2024-09-21T10:52:50.350 INFO:journalctl@ceph.osd.2.smithi123.stdout:Sep 21 10:52:50 smithi123 podman[94360]: 2024-09-21 10:52:50.077844153 +0000 UTC m=+0.628903960 container remove 647aec759e3c2f1473b2fea786ab91eee17aff7e2f503e41018d8b226c11ffd2 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240731, GIT_CLEAN=True) 2024-09-21T10:52:50.699 INFO:journalctl@ceph.osd.2.smithi123.stdout:Sep 21 10:52:50 smithi123 podman[94449]: 2024-09-21 10:52:50.338259944 +0000 UTC m=+0.016390685 image pull 8f4d013db793795fef09ff5edf96e6b42ba8cd2c9158b2f0b7a5d7c1e657c063 quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472 2024-09-21T10:52:50.699 INFO:journalctl@ceph.osd.2.smithi123.stdout:Sep 21 10:52:50 smithi123 podman[94449]: 2024-09-21 10:52:50.459108708 +0000 UTC m=+0.137239449 container create 9a894153a1e9bf4ed588a0ae4bb5a89a9f6cc1738406a2639bd96db980a99cc7 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-2-deactivate, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, org.label-schema.build-date=20240731, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T10:52:50.699 INFO:journalctl@ceph.osd.2.smithi123.stdout:Sep 21 10:52:50 smithi123 podman[94449]: 2024-09-21 10:52:50.698088641 +0000 UTC m=+0.376219383 container init 9a894153a1e9bf4ed588a0ae4bb5a89a9f6cc1738406a2639bd96db980a99cc7 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-2-deactivate, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_CLEAN=True, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T10:52:51.101 INFO:journalctl@ceph.osd.2.smithi123.stdout:Sep 21 10:52:50 smithi123 podman[94449]: 2024-09-21 10:52:50.701643884 +0000 UTC m=+0.379774628 container start 9a894153a1e9bf4ed588a0ae4bb5a89a9f6cc1738406a2639bd96db980a99cc7 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-2-deactivate, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T10:52:51.101 INFO:journalctl@ceph.osd.2.smithi123.stdout:Sep 21 10:52:50 smithi123 podman[94449]: 2024-09-21 10:52:50.797798705 +0000 UTC m=+0.475929464 container attach 9a894153a1e9bf4ed588a0ae4bb5a89a9f6cc1738406a2639bd96db980a99cc7 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-2-deactivate, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731) 2024-09-21T10:52:51.101 INFO:journalctl@ceph.osd.2.smithi123.stdout:Sep 21 10:52:50 smithi123 podman[94449]: 2024-09-21 10:52:50.891903965 +0000 UTC m=+0.570034710 container died 9a894153a1e9bf4ed588a0ae4bb5a89a9f6cc1738406a2639bd96db980a99cc7 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-2-deactivate, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T10:52:51.466 DEBUG:teuthology.orchestra.run.smithi123:> sudo pkill -f 'journalctl -f -n 0 -u ceph-3ccd15e6-7804-11ef-baf6-efdc52797490@osd.2.service' 2024-09-21T10:52:51.505 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T10:52:51.505 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-09-21T10:52:51.505 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-09-21T10:52:51.505 DEBUG:teuthology.orchestra.run.smithi123:> sudo systemctl stop ceph-3ccd15e6-7804-11ef-baf6-efdc52797490@osd.3 2024-09-21T10:52:51.831 INFO:journalctl@ceph.osd.3.smithi123.stdout:Sep 21 10:52:51 smithi123 systemd[1]: Stopping Ceph osd.3 for 3ccd15e6-7804-11ef-baf6-efdc52797490... 2024-09-21T10:52:52.100 INFO:journalctl@ceph.osd.3.smithi123.stdout:Sep 21 10:52:51 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-3[76185]: 2024-09-21T10:52:51.828+0000 7ffbf9974640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:52:52.101 INFO:journalctl@ceph.osd.3.smithi123.stdout:Sep 21 10:52:51 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-3[76185]: 2024-09-21T10:52:51.828+0000 7ffbf9974640 -1 osd.3 174 *** Got signal Terminated *** 2024-09-21T10:52:52.101 INFO:journalctl@ceph.osd.3.smithi123.stdout:Sep 21 10:52:51 smithi123 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-3[76185]: 2024-09-21T10:52:51.828+0000 7ffbf9974640 -1 osd.3 174 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T10:52:55.152 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:52:55 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-4[53269]: 2024-09-21T10:52:55.052+0000 7fb5112c4640 -1 osd.4 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:31.966505+0000 front 2024-09-21T10:52:31.966554+0000 (oldest deadline 2024-09-21T10:52:54.866510+0000) 2024-09-21T10:52:55.152 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:52:54 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:52:54.898+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:30.897816+0000 front 2024-09-21T10:52:30.897736+0000 (oldest deadline 2024-09-21T10:52:54.397710+0000) 2024-09-21T10:52:55.882 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:52:55 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-5[55349]: 2024-09-21T10:52:55.621+0000 7f255a6d0640 -1 osd.5 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:31.372631+0000 front 2024-09-21T10:52:31.372464+0000 (oldest deadline 2024-09-21T10:52:55.472351+0000) 2024-09-21T10:52:56.373 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:52:55 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:52:55.880+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:30.897816+0000 front 2024-09-21T10:52:30.897736+0000 (oldest deadline 2024-09-21T10:52:54.397710+0000) 2024-09-21T10:52:56.373 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:52:56 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-4[53269]: 2024-09-21T10:52:56.069+0000 7fb5112c4640 -1 osd.4 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:31.966505+0000 front 2024-09-21T10:52:31.966554+0000 (oldest deadline 2024-09-21T10:52:54.866510+0000) 2024-09-21T10:52:57.030 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:52:56 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-5[55349]: 2024-09-21T10:52:56.658+0000 7f255a6d0640 -1 osd.5 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:31.372631+0000 front 2024-09-21T10:52:31.372464+0000 (oldest deadline 2024-09-21T10:52:55.472351+0000) 2024-09-21T10:52:57.030 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:52:56 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:52:56.908+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:30.897816+0000 front 2024-09-21T10:52:30.897736+0000 (oldest deadline 2024-09-21T10:52:54.397710+0000) 2024-09-21T10:52:57.373 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:52:57 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-4[53269]: 2024-09-21T10:52:57.027+0000 7fb5112c4640 -1 osd.4 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:31.966505+0000 front 2024-09-21T10:52:31.966554+0000 (oldest deadline 2024-09-21T10:52:54.866510+0000) 2024-09-21T10:52:57.741 INFO:journalctl@ceph.osd.3.smithi123.stdout:Sep 21 10:52:57 smithi123 podman[94550]: 2024-09-21 10:52:57.427501228 +0000 UTC m=+5.699494701 container died 7a082fcad8e897d96dd27cc72399d5e23e119699d6490ce9300e777f4eb6ad80 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-3, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=) 2024-09-21T10:52:58.000 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:52:57 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-5[55349]: 2024-09-21T10:52:57.677+0000 7f255a6d0640 -1 osd.5 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:31.372631+0000 front 2024-09-21T10:52:31.372464+0000 (oldest deadline 2024-09-21T10:52:55.472351+0000) 2024-09-21T10:52:58.000 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:52:57 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:52:57.905+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:30.897816+0000 front 2024-09-21T10:52:30.897736+0000 (oldest deadline 2024-09-21T10:52:54.397710+0000) 2024-09-21T10:52:58.101 INFO:journalctl@ceph.osd.3.smithi123.stdout:Sep 21 10:52:57 smithi123 podman[94550]: 2024-09-21 10:52:57.820465306 +0000 UTC m=+6.092458703 container cleanup 7a082fcad8e897d96dd27cc72399d5e23e119699d6490ce9300e777f4eb6ad80 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-3, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, org.label-schema.vendor=CentOS) 2024-09-21T10:52:58.101 INFO:journalctl@ceph.osd.3.smithi123.stdout:Sep 21 10:52:57 smithi123 bash[94550]: ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-3 2024-09-21T10:52:58.101 INFO:journalctl@ceph.osd.3.smithi123.stdout:Sep 21 10:52:57 smithi123 podman[94582]: 2024-09-21 10:52:57.988333783 +0000 UTC m=+0.552529835 container remove 7a082fcad8e897d96dd27cc72399d5e23e119699d6490ce9300e777f4eb6ad80 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-3, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2) 2024-09-21T10:52:58.367 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:52:57 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-4[53269]: 2024-09-21T10:52:57.998+0000 7fb5112c4640 -1 osd.4 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:31.966505+0000 front 2024-09-21T10:52:31.966554+0000 (oldest deadline 2024-09-21T10:52:54.866510+0000) 2024-09-21T10:52:58.623 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:52:58 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6[57429]: 2024-09-21T10:52:58.364+0000 7fc564743640 -1 osd.6 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:28.588077+0000 front 2024-09-21T10:52:28.588155+0000 (oldest deadline 2024-09-21T10:52:57.388747+0000) 2024-09-21T10:52:58.759 INFO:journalctl@ceph.osd.3.smithi123.stdout:Sep 21 10:52:58 smithi123 podman[94671]: 2024-09-21 10:52:58.334853732 +0000 UTC m=+0.016155547 image pull 8f4d013db793795fef09ff5edf96e6b42ba8cd2c9158b2f0b7a5d7c1e657c063 quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472 2024-09-21T10:52:58.760 INFO:journalctl@ceph.osd.3.smithi123.stdout:Sep 21 10:52:58 smithi123 podman[94671]: 2024-09-21 10:52:58.464138109 +0000 UTC m=+0.145439914 container create c2e83359406f3f3821c572c300d4c8e0341ca5d4780e59ebf7e3e95abddf1c72 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-3-deactivate, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True) 2024-09-21T10:52:58.760 INFO:journalctl@ceph.osd.3.smithi123.stdout:Sep 21 10:52:58 smithi123 podman[94671]: 2024-09-21 10:52:58.675184544 +0000 UTC m=+0.356486354 container init c2e83359406f3f3821c572c300d4c8e0341ca5d4780e59ebf7e3e95abddf1c72 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-3-deactivate, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3) 2024-09-21T10:52:58.760 INFO:journalctl@ceph.osd.3.smithi123.stdout:Sep 21 10:52:58 smithi123 podman[94671]: 2024-09-21 10:52:58.678664611 +0000 UTC m=+0.359966419 container start c2e83359406f3f3821c572c300d4c8e0341ca5d4780e59ebf7e3e95abddf1c72 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-3-deactivate, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T10:52:58.760 INFO:journalctl@ceph.osd.3.smithi123.stdout:Sep 21 10:52:58 smithi123 podman[94671]: 2024-09-21 10:52:58.758383946 +0000 UTC m=+0.439685763 container attach c2e83359406f3f3821c572c300d4c8e0341ca5d4780e59ebf7e3e95abddf1c72 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-3-deactivate, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-09-21T10:52:58.967 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:52:58 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-5[55349]: 2024-09-21T10:52:58.680+0000 7f255a6d0640 -1 osd.5 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:31.372631+0000 front 2024-09-21T10:52:31.372464+0000 (oldest deadline 2024-09-21T10:52:55.472351+0000) 2024-09-21T10:52:58.967 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:52:58 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:52:58.901+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:30.897816+0000 front 2024-09-21T10:52:30.897736+0000 (oldest deadline 2024-09-21T10:52:54.397710+0000) 2024-09-21T10:52:59.100 INFO:journalctl@ceph.osd.3.smithi123.stdout:Sep 21 10:52:58 smithi123 podman[94671]: 2024-09-21 10:52:58.867651114 +0000 UTC m=+0.548952919 container died c2e83359406f3f3821c572c300d4c8e0341ca5d4780e59ebf7e3e95abddf1c72 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-3-deactivate, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T10:52:59.373 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:52:58 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-4[53269]: 2024-09-21T10:52:58.964+0000 7fb5112c4640 -1 osd.4 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:31.966505+0000 front 2024-09-21T10:52:31.966554+0000 (oldest deadline 2024-09-21T10:52:54.866510+0000) 2024-09-21T10:52:59.462 DEBUG:teuthology.orchestra.run.smithi123:> sudo pkill -f 'journalctl -f -n 0 -u ceph-3ccd15e6-7804-11ef-baf6-efdc52797490@osd.3.service' 2024-09-21T10:52:59.499 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T10:52:59.500 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-09-21T10:52:59.500 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-09-21T10:52:59.500 DEBUG:teuthology.orchestra.run.smithi175:> sudo systemctl stop ceph-3ccd15e6-7804-11ef-baf6-efdc52797490@osd.4 2024-09-21T10:52:59.732 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:52:59 smithi175 systemd[1]: Stopping Ceph osd.4 for 3ccd15e6-7804-11ef-baf6-efdc52797490... 2024-09-21T10:52:59.732 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:52:59 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6[57429]: 2024-09-21T10:52:59.379+0000 7fc564743640 -1 osd.6 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:28.588077+0000 front 2024-09-21T10:52:28.588155+0000 (oldest deadline 2024-09-21T10:52:57.388747+0000) 2024-09-21T10:53:00.011 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:52:59 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-5[55349]: 2024-09-21T10:52:59.729+0000 7f255a6d0640 -1 osd.5 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:31.372631+0000 front 2024-09-21T10:52:31.372464+0000 (oldest deadline 2024-09-21T10:52:55.472351+0000) 2024-09-21T10:53:00.011 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:52:59 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:52:59.874+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:30.897816+0000 front 2024-09-21T10:52:30.897736+0000 (oldest deadline 2024-09-21T10:52:54.397710+0000) 2024-09-21T10:53:00.011 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:52:59 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-4[53269]: 2024-09-21T10:52:59.813+0000 7fb514caf640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:53:00.011 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:52:59 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-4[53269]: 2024-09-21T10:52:59.813+0000 7fb514caf640 -1 osd.4 174 *** Got signal Terminated *** 2024-09-21T10:53:00.012 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:52:59 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-4[53269]: 2024-09-21T10:52:59.813+0000 7fb514caf640 -1 osd.4 174 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T10:53:00.012 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:53:00 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-4[53269]: 2024-09-21T10:53:00.008+0000 7fb5112c4640 -1 osd.4 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:31.966505+0000 front 2024-09-21T10:52:31.966554+0000 (oldest deadline 2024-09-21T10:52:54.866510+0000) 2024-09-21T10:53:00.745 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:53:00 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6[57429]: 2024-09-21T10:53:00.418+0000 7fc564743640 -1 osd.6 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:28.588077+0000 front 2024-09-21T10:52:28.588155+0000 (oldest deadline 2024-09-21T10:52:57.388747+0000) 2024-09-21T10:53:01.004 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:00 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:53:00.860+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:30.897816+0000 front 2024-09-21T10:52:30.897736+0000 (oldest deadline 2024-09-21T10:52:54.397710+0000) 2024-09-21T10:53:01.004 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:53:00 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-5[55349]: 2024-09-21T10:53:00.743+0000 7f255a6d0640 -1 osd.5 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:31.372631+0000 front 2024-09-21T10:52:31.372464+0000 (oldest deadline 2024-09-21T10:52:55.472351+0000) 2024-09-21T10:53:01.373 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:53:01 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-4[53269]: 2024-09-21T10:53:01.002+0000 7fb5112c4640 -1 osd.4 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:31.966505+0000 front 2024-09-21T10:52:31.966554+0000 (oldest deadline 2024-09-21T10:52:54.866510+0000) 2024-09-21T10:53:01.765 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:53:01 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6[57429]: 2024-09-21T10:53:01.372+0000 7fc564743640 -1 osd.6 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:28.588077+0000 front 2024-09-21T10:52:28.588155+0000 (oldest deadline 2024-09-21T10:52:57.388747+0000) 2024-09-21T10:53:02.048 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:53:01 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-5[55349]: 2024-09-21T10:53:01.763+0000 7f255a6d0640 -1 osd.5 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:31.372631+0000 front 2024-09-21T10:52:31.372464+0000 (oldest deadline 2024-09-21T10:52:55.472351+0000) 2024-09-21T10:53:02.048 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:01 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:53:01.834+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:30.897816+0000 front 2024-09-21T10:52:30.897736+0000 (oldest deadline 2024-09-21T10:52:54.397710+0000) 2024-09-21T10:53:02.327 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:53:02 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-4[53269]: 2024-09-21T10:53:02.046+0000 7fb5112c4640 -1 osd.4 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:31.966505+0000 front 2024-09-21T10:52:31.966554+0000 (oldest deadline 2024-09-21T10:52:54.866510+0000) 2024-09-21T10:53:02.623 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:53:02 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6[57429]: 2024-09-21T10:53:02.325+0000 7fc564743640 -1 osd.6 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:28.588077+0000 front 2024-09-21T10:52:28.588155+0000 (oldest deadline 2024-09-21T10:52:57.388747+0000) 2024-09-21T10:53:03.043 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:02 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:53:02.818+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:30.897816+0000 front 2024-09-21T10:52:30.897736+0000 (oldest deadline 2024-09-21T10:52:54.397710+0000) 2024-09-21T10:53:03.043 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:53:02 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-5[55349]: 2024-09-21T10:53:02.751+0000 7f255a6d0640 -1 osd.5 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:31.372631+0000 front 2024-09-21T10:52:31.372464+0000 (oldest deadline 2024-09-21T10:52:55.472351+0000) 2024-09-21T10:53:03.306 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:53:03 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-4[53269]: 2024-09-21T10:53:03.041+0000 7fb5112c4640 -1 osd.4 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:31.966505+0000 front 2024-09-21T10:52:31.966554+0000 (oldest deadline 2024-09-21T10:52:54.866510+0000) 2024-09-21T10:53:03.306 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:53:03 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-4[53269]: 2024-09-21T10:53:03.041+0000 7fb5112c4640 -1 osd.4 174 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-09-21T10:52:37.667982+0000 front 2024-09-21T10:52:37.668026+0000 (oldest deadline 2024-09-21T10:53:02.367936+0000) 2024-09-21T10:53:03.623 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:53:03 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6[57429]: 2024-09-21T10:53:03.304+0000 7fc564743640 -1 osd.6 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:28.588077+0000 front 2024-09-21T10:52:28.588155+0000 (oldest deadline 2024-09-21T10:52:57.388747+0000) 2024-09-21T10:53:03.624 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:53:03 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6[57429]: 2024-09-21T10:53:03.304+0000 7fc564743640 -1 osd.6 174 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-09-21T10:52:40.190381+0000 front 2024-09-21T10:52:40.190460+0000 (oldest deadline 2024-09-21T10:53:02.390975+0000) 2024-09-21T10:53:04.074 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:03 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:53:03.856+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:30.897816+0000 front 2024-09-21T10:52:30.897736+0000 (oldest deadline 2024-09-21T10:52:54.397710+0000) 2024-09-21T10:53:04.074 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:53:03 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-5[55349]: 2024-09-21T10:53:03.747+0000 7f255a6d0640 -1 osd.5 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:31.372631+0000 front 2024-09-21T10:52:31.372464+0000 (oldest deadline 2024-09-21T10:52:55.472351+0000) 2024-09-21T10:53:04.075 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:53:03 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-5[55349]: 2024-09-21T10:53:03.747+0000 7f255a6d0640 -1 osd.5 174 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-09-21T10:52:39.974319+0000 front 2024-09-21T10:52:39.974425+0000 (oldest deadline 2024-09-21T10:53:02.874198+0000) 2024-09-21T10:53:04.373 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:53:04 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-4[53269]: 2024-09-21T10:53:04.072+0000 7fb5112c4640 -1 osd.4 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:31.966505+0000 front 2024-09-21T10:52:31.966554+0000 (oldest deadline 2024-09-21T10:52:54.866510+0000) 2024-09-21T10:53:04.373 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:53:04 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-4[53269]: 2024-09-21T10:53:04.072+0000 7fb5112c4640 -1 osd.4 174 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-09-21T10:52:37.667982+0000 front 2024-09-21T10:52:37.668026+0000 (oldest deadline 2024-09-21T10:53:02.367936+0000) 2024-09-21T10:53:04.373 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:53:04 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6[57429]: 2024-09-21T10:53:04.297+0000 7fc564743640 -1 osd.6 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:28.588077+0000 front 2024-09-21T10:52:28.588155+0000 (oldest deadline 2024-09-21T10:52:57.388747+0000) 2024-09-21T10:53:04.373 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:53:04 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6[57429]: 2024-09-21T10:53:04.297+0000 7fc564743640 -1 osd.6 174 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-09-21T10:52:40.190381+0000 front 2024-09-21T10:52:40.190460+0000 (oldest deadline 2024-09-21T10:53:02.390975+0000) 2024-09-21T10:53:05.123 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:04 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:53:04.824+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:30.897816+0000 front 2024-09-21T10:52:30.897736+0000 (oldest deadline 2024-09-21T10:52:54.397710+0000) 2024-09-21T10:53:05.123 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:53:04 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-5[55349]: 2024-09-21T10:53:04.771+0000 7f255a6d0640 -1 osd.5 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:31.372631+0000 front 2024-09-21T10:52:31.372464+0000 (oldest deadline 2024-09-21T10:52:55.472351+0000) 2024-09-21T10:53:05.124 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:53:04 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-5[55349]: 2024-09-21T10:53:04.771+0000 7f255a6d0640 -1 osd.5 174 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-09-21T10:52:39.974319+0000 front 2024-09-21T10:52:39.974425+0000 (oldest deadline 2024-09-21T10:53:02.874198+0000) 2024-09-21T10:53:05.532 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:53:05 smithi175 podman[68561]: 2024-09-21 10:53:05.251696107 +0000 UTC m=+5.558094521 container died b650f72d10f046eaada2771b4eb808f48067cdcc25727c8d706f941cb70e0bea (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-4, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2) 2024-09-21T10:53:05.532 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:53:05 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6[57429]: 2024-09-21T10:53:05.274+0000 7fc564743640 -1 osd.6 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:28.588077+0000 front 2024-09-21T10:52:28.588155+0000 (oldest deadline 2024-09-21T10:52:57.388747+0000) 2024-09-21T10:53:05.532 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:53:05 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6[57429]: 2024-09-21T10:53:05.274+0000 7fc564743640 -1 osd.6 174 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-09-21T10:52:40.190381+0000 front 2024-09-21T10:52:40.190460+0000 (oldest deadline 2024-09-21T10:53:02.390975+0000) 2024-09-21T10:53:05.783 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:53:05 smithi175 podman[68561]: 2024-09-21 10:53:05.738008151 +0000 UTC m=+6.044406591 container cleanup b650f72d10f046eaada2771b4eb808f48067cdcc25727c8d706f941cb70e0bea (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-4, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T10:53:05.783 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:53:05 smithi175 bash[68561]: ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-4 2024-09-21T10:53:06.123 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:53:05 smithi175 podman[68576]: 2024-09-21 10:53:05.997869627 +0000 UTC m=+0.746620849 container remove b650f72d10f046eaada2771b4eb808f48067cdcc25727c8d706f941cb70e0bea (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-4, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-09-21T10:53:06.123 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:53:05 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-5[55349]: 2024-09-21T10:53:05.781+0000 7f255a6d0640 -1 osd.5 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:31.372631+0000 front 2024-09-21T10:52:31.372464+0000 (oldest deadline 2024-09-21T10:52:55.472351+0000) 2024-09-21T10:53:06.124 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:53:05 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-5[55349]: 2024-09-21T10:53:05.781+0000 7f255a6d0640 -1 osd.5 174 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-09-21T10:52:39.974319+0000 front 2024-09-21T10:52:39.974425+0000 (oldest deadline 2024-09-21T10:53:02.874198+0000) 2024-09-21T10:53:06.124 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:05 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:53:05.840+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:30.897816+0000 front 2024-09-21T10:52:30.897736+0000 (oldest deadline 2024-09-21T10:52:54.397710+0000) 2024-09-21T10:53:06.124 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:05 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:53:05.840+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-09-21T10:52:39.698984+0000 front 2024-09-21T10:52:39.698943+0000 (oldest deadline 2024-09-21T10:53:05.598730+0000) 2024-09-21T10:53:06.611 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:53:06 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6[57429]: 2024-09-21T10:53:06.251+0000 7fc564743640 -1 osd.6 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:28.588077+0000 front 2024-09-21T10:52:28.588155+0000 (oldest deadline 2024-09-21T10:52:57.388747+0000) 2024-09-21T10:53:06.611 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:53:06 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6[57429]: 2024-09-21T10:53:06.251+0000 7fc564743640 -1 osd.6 174 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-09-21T10:52:40.190381+0000 front 2024-09-21T10:52:40.190460+0000 (oldest deadline 2024-09-21T10:53:02.390975+0000) 2024-09-21T10:53:06.611 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:53:06 smithi175 podman[68665]: 2024-09-21 10:53:06.26297841 +0000 UTC m=+0.016411926 image pull 8f4d013db793795fef09ff5edf96e6b42ba8cd2c9158b2f0b7a5d7c1e657c063 quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472 2024-09-21T10:53:06.611 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:53:06 smithi175 podman[68665]: 2024-09-21 10:53:06.373635796 +0000 UTC m=+0.127069311 container create b5955ca8f128983027bcfaec568bb54e1d2cb2b647a3b6e032381d75eb96e94c (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-4-deactivate, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, ceph=True, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-09-21T10:53:06.873 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:53:06 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-5[55349]: 2024-09-21T10:53:06.813+0000 7f255a6d0640 -1 osd.5 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:31.372631+0000 front 2024-09-21T10:52:31.372464+0000 (oldest deadline 2024-09-21T10:52:55.472351+0000) 2024-09-21T10:53:06.874 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:53:06 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-5[55349]: 2024-09-21T10:53:06.813+0000 7f255a6d0640 -1 osd.5 174 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-09-21T10:52:39.974319+0000 front 2024-09-21T10:52:39.974425+0000 (oldest deadline 2024-09-21T10:53:02.874198+0000) 2024-09-21T10:53:06.874 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:06 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:53:06.858+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:30.897816+0000 front 2024-09-21T10:52:30.897736+0000 (oldest deadline 2024-09-21T10:52:54.397710+0000) 2024-09-21T10:53:06.874 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:06 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:53:06.858+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-09-21T10:52:39.698984+0000 front 2024-09-21T10:52:39.698943+0000 (oldest deadline 2024-09-21T10:53:05.598730+0000) 2024-09-21T10:53:06.874 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:53:06 smithi175 podman[68665]: 2024-09-21 10:53:06.60946016 +0000 UTC m=+0.362893691 container init b5955ca8f128983027bcfaec568bb54e1d2cb2b647a3b6e032381d75eb96e94c (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-4-deactivate, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-09-21T10:53:06.875 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:53:06 smithi175 podman[68665]: 2024-09-21 10:53:06.613169901 +0000 UTC m=+0.366603416 container start b5955ca8f128983027bcfaec568bb54e1d2cb2b647a3b6e032381d75eb96e94c (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-4-deactivate, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T10:53:06.875 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:53:06 smithi175 podman[68665]: 2024-09-21 10:53:06.700978366 +0000 UTC m=+0.454411894 container attach b5955ca8f128983027bcfaec568bb54e1d2cb2b647a3b6e032381d75eb96e94c (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-4-deactivate, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T10:53:06.875 INFO:journalctl@ceph.osd.4.smithi175.stdout:Sep 21 10:53:06 smithi175 podman[68665]: 2024-09-21 10:53:06.802617833 +0000 UTC m=+0.556051356 container died b5955ca8f128983027bcfaec568bb54e1d2cb2b647a3b6e032381d75eb96e94c (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-4-deactivate, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T10:53:07.373 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:53:07 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6[57429]: 2024-09-21T10:53:07.281+0000 7fc564743640 -1 osd.6 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:28.588077+0000 front 2024-09-21T10:52:28.588155+0000 (oldest deadline 2024-09-21T10:52:57.388747+0000) 2024-09-21T10:53:07.373 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:53:07 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6[57429]: 2024-09-21T10:53:07.281+0000 7fc564743640 -1 osd.6 174 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-09-21T10:52:40.190381+0000 front 2024-09-21T10:52:40.190460+0000 (oldest deadline 2024-09-21T10:53:02.390975+0000) 2024-09-21T10:53:07.433 DEBUG:teuthology.orchestra.run.smithi175:> sudo pkill -f 'journalctl -f -n 0 -u ceph-3ccd15e6-7804-11ef-baf6-efdc52797490@osd.4.service' 2024-09-21T10:53:07.474 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T10:53:07.474 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-09-21T10:53:07.475 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-09-21T10:53:07.475 DEBUG:teuthology.orchestra.run.smithi175:> sudo systemctl stop ceph-3ccd15e6-7804-11ef-baf6-efdc52797490@osd.5 2024-09-21T10:53:07.667 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:53:07 smithi175 systemd[1]: Stopping Ceph osd.5 for 3ccd15e6-7804-11ef-baf6-efdc52797490... 2024-09-21T10:53:08.123 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:07 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:53:07.843+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:30.897816+0000 front 2024-09-21T10:52:30.897736+0000 (oldest deadline 2024-09-21T10:52:54.397710+0000) 2024-09-21T10:53:08.124 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:07 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:53:07.843+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-09-21T10:52:39.698984+0000 front 2024-09-21T10:52:39.698943+0000 (oldest deadline 2024-09-21T10:53:05.598730+0000) 2024-09-21T10:53:08.124 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:53:07 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-5[55349]: 2024-09-21T10:53:07.824+0000 7f255e0bb640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:53:08.124 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:53:07 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-5[55349]: 2024-09-21T10:53:07.824+0000 7f255e0bb640 -1 osd.5 174 *** Got signal Terminated *** 2024-09-21T10:53:08.124 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:53:07 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-5[55349]: 2024-09-21T10:53:07.824+0000 7f255e0bb640 -1 osd.5 174 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T10:53:08.124 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:53:07 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-5[55349]: 2024-09-21T10:53:07.836+0000 7f255a6d0640 -1 osd.5 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:31.372631+0000 front 2024-09-21T10:52:31.372464+0000 (oldest deadline 2024-09-21T10:52:55.472351+0000) 2024-09-21T10:53:08.124 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:53:07 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-5[55349]: 2024-09-21T10:53:07.836+0000 7f255a6d0640 -1 osd.5 174 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-09-21T10:52:39.974319+0000 front 2024-09-21T10:52:39.974425+0000 (oldest deadline 2024-09-21T10:53:02.874198+0000) 2024-09-21T10:53:08.621 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:53:08 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6[57429]: 2024-09-21T10:53:08.329+0000 7fc564743640 -1 osd.6 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:28.588077+0000 front 2024-09-21T10:52:28.588155+0000 (oldest deadline 2024-09-21T10:52:57.388747+0000) 2024-09-21T10:53:08.621 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:53:08 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6[57429]: 2024-09-21T10:53:08.329+0000 7fc564743640 -1 osd.6 174 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-09-21T10:52:40.190381+0000 front 2024-09-21T10:52:40.190460+0000 (oldest deadline 2024-09-21T10:53:02.390975+0000) 2024-09-21T10:53:08.873 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:53:08 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-5[55349]: 2024-09-21T10:53:08.828+0000 7f255a6d0640 -1 osd.5 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:31.372631+0000 front 2024-09-21T10:52:31.372464+0000 (oldest deadline 2024-09-21T10:52:55.472351+0000) 2024-09-21T10:53:08.873 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:53:08 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-5[55349]: 2024-09-21T10:53:08.828+0000 7f255a6d0640 -1 osd.5 174 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-09-21T10:52:39.974319+0000 front 2024-09-21T10:52:39.974425+0000 (oldest deadline 2024-09-21T10:53:02.874198+0000) 2024-09-21T10:53:08.874 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:08 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:53:08.797+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:30.897816+0000 front 2024-09-21T10:52:30.897736+0000 (oldest deadline 2024-09-21T10:52:54.397710+0000) 2024-09-21T10:53:08.874 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:08 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:53:08.797+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-09-21T10:52:39.698984+0000 front 2024-09-21T10:52:39.698943+0000 (oldest deadline 2024-09-21T10:53:05.598730+0000) 2024-09-21T10:53:09.623 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:53:09 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6[57429]: 2024-09-21T10:53:09.291+0000 7fc564743640 -1 osd.6 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:28.588077+0000 front 2024-09-21T10:52:28.588155+0000 (oldest deadline 2024-09-21T10:52:57.388747+0000) 2024-09-21T10:53:09.623 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:53:09 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6[57429]: 2024-09-21T10:53:09.291+0000 7fc564743640 -1 osd.6 174 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-09-21T10:52:40.190381+0000 front 2024-09-21T10:52:40.190460+0000 (oldest deadline 2024-09-21T10:53:02.390975+0000) 2024-09-21T10:53:10.123 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:53:09 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-5[55349]: 2024-09-21T10:53:09.844+0000 7f255a6d0640 -1 osd.5 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:31.372631+0000 front 2024-09-21T10:52:31.372464+0000 (oldest deadline 2024-09-21T10:52:55.472351+0000) 2024-09-21T10:53:10.123 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:53:09 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-5[55349]: 2024-09-21T10:53:09.844+0000 7f255a6d0640 -1 osd.5 174 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-09-21T10:52:39.974319+0000 front 2024-09-21T10:52:39.974425+0000 (oldest deadline 2024-09-21T10:53:02.874198+0000) 2024-09-21T10:53:10.124 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:09 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:53:09.793+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:30.897816+0000 front 2024-09-21T10:52:30.897736+0000 (oldest deadline 2024-09-21T10:52:54.397710+0000) 2024-09-21T10:53:10.124 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:09 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:53:09.793+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-09-21T10:52:39.698984+0000 front 2024-09-21T10:52:39.698943+0000 (oldest deadline 2024-09-21T10:53:05.598730+0000) 2024-09-21T10:53:10.623 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:53:10 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6[57429]: 2024-09-21T10:53:10.313+0000 7fc564743640 -1 osd.6 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:28.588077+0000 front 2024-09-21T10:52:28.588155+0000 (oldest deadline 2024-09-21T10:52:57.388747+0000) 2024-09-21T10:53:10.623 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:53:10 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6[57429]: 2024-09-21T10:53:10.313+0000 7fc564743640 -1 osd.6 174 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-09-21T10:52:40.190381+0000 front 2024-09-21T10:52:40.190460+0000 (oldest deadline 2024-09-21T10:53:02.390975+0000) 2024-09-21T10:53:11.123 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:53:10 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-5[55349]: 2024-09-21T10:53:10.836+0000 7f255a6d0640 -1 osd.5 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:31.372631+0000 front 2024-09-21T10:52:31.372464+0000 (oldest deadline 2024-09-21T10:52:55.472351+0000) 2024-09-21T10:53:11.123 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:53:10 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-5[55349]: 2024-09-21T10:53:10.836+0000 7f255a6d0640 -1 osd.5 174 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-09-21T10:52:39.974319+0000 front 2024-09-21T10:52:39.974425+0000 (oldest deadline 2024-09-21T10:53:02.874198+0000) 2024-09-21T10:53:11.124 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:10 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:53:10.765+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:30.897816+0000 front 2024-09-21T10:52:30.897736+0000 (oldest deadline 2024-09-21T10:52:54.397710+0000) 2024-09-21T10:53:11.124 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:10 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:53:10.765+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-09-21T10:52:39.698984+0000 front 2024-09-21T10:52:39.698943+0000 (oldest deadline 2024-09-21T10:53:05.598730+0000) 2024-09-21T10:53:11.623 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:53:11 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6[57429]: 2024-09-21T10:53:11.302+0000 7fc564743640 -1 osd.6 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:28.588077+0000 front 2024-09-21T10:52:28.588155+0000 (oldest deadline 2024-09-21T10:52:57.388747+0000) 2024-09-21T10:53:11.623 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:53:11 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6[57429]: 2024-09-21T10:53:11.302+0000 7fc564743640 -1 osd.6 174 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-09-21T10:52:40.190381+0000 front 2024-09-21T10:52:40.190460+0000 (oldest deadline 2024-09-21T10:53:02.390975+0000) 2024-09-21T10:53:12.123 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:53:11 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-5[55349]: 2024-09-21T10:53:11.804+0000 7f255a6d0640 -1 osd.5 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:31.372631+0000 front 2024-09-21T10:52:31.372464+0000 (oldest deadline 2024-09-21T10:52:55.472351+0000) 2024-09-21T10:53:12.123 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:53:11 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-5[55349]: 2024-09-21T10:53:11.804+0000 7f255a6d0640 -1 osd.5 174 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-09-21T10:52:39.974319+0000 front 2024-09-21T10:52:39.974425+0000 (oldest deadline 2024-09-21T10:53:02.874198+0000) 2024-09-21T10:53:12.123 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:11 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:53:11.716+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:30.897816+0000 front 2024-09-21T10:52:30.897736+0000 (oldest deadline 2024-09-21T10:52:54.397710+0000) 2024-09-21T10:53:12.124 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:11 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:53:11.716+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-09-21T10:52:39.698984+0000 front 2024-09-21T10:52:39.698943+0000 (oldest deadline 2024-09-21T10:53:05.598730+0000) 2024-09-21T10:53:12.124 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:11 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:53:11.716+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.123:6822 osd.2 since back 2024-09-21T10:52:45.599268+0000 front 2024-09-21T10:52:45.599206+0000 (oldest deadline 2024-09-21T10:53:10.899146+0000) 2024-09-21T10:53:12.623 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:53:12 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6[57429]: 2024-09-21T10:53:12.290+0000 7fc564743640 -1 osd.6 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:28.588077+0000 front 2024-09-21T10:52:28.588155+0000 (oldest deadline 2024-09-21T10:52:57.388747+0000) 2024-09-21T10:53:12.623 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:53:12 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6[57429]: 2024-09-21T10:53:12.290+0000 7fc564743640 -1 osd.6 174 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-09-21T10:52:40.190381+0000 front 2024-09-21T10:52:40.190460+0000 (oldest deadline 2024-09-21T10:53:02.390975+0000) 2024-09-21T10:53:13.123 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:53:12 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-5[55349]: 2024-09-21T10:53:12.824+0000 7f255a6d0640 -1 osd.5 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:31.372631+0000 front 2024-09-21T10:52:31.372464+0000 (oldest deadline 2024-09-21T10:52:55.472351+0000) 2024-09-21T10:53:13.123 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:53:12 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-5[55349]: 2024-09-21T10:53:12.824+0000 7f255a6d0640 -1 osd.5 174 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-09-21T10:52:39.974319+0000 front 2024-09-21T10:52:39.974425+0000 (oldest deadline 2024-09-21T10:53:02.874198+0000) 2024-09-21T10:53:13.124 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:12 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:53:12.698+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:30.897816+0000 front 2024-09-21T10:52:30.897736+0000 (oldest deadline 2024-09-21T10:52:54.397710+0000) 2024-09-21T10:53:13.124 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:12 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:53:12.698+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-09-21T10:52:39.698984+0000 front 2024-09-21T10:52:39.698943+0000 (oldest deadline 2024-09-21T10:53:05.598730+0000) 2024-09-21T10:53:13.124 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:12 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:53:12.698+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.123:6822 osd.2 since back 2024-09-21T10:52:45.599268+0000 front 2024-09-21T10:52:45.599206+0000 (oldest deadline 2024-09-21T10:53:10.899146+0000) 2024-09-21T10:53:13.559 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:53:13 smithi175 podman[68768]: 2024-09-21 10:53:13.263548687 +0000 UTC m=+5.556145652 container died 8199a0b362df6545a0b031557ce2948aa33d9cb647d1549c9861e25558caf68f (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-5, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0) 2024-09-21T10:53:13.560 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:53:13 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6[57429]: 2024-09-21T10:53:13.297+0000 7fc564743640 -1 osd.6 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:28.588077+0000 front 2024-09-21T10:52:28.588155+0000 (oldest deadline 2024-09-21T10:52:57.388747+0000) 2024-09-21T10:53:13.560 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:53:13 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6[57429]: 2024-09-21T10:53:13.297+0000 7fc564743640 -1 osd.6 174 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-09-21T10:52:40.190381+0000 front 2024-09-21T10:52:40.190460+0000 (oldest deadline 2024-09-21T10:53:02.390975+0000) 2024-09-21T10:53:13.873 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:53:13 smithi175 podman[68768]: 2024-09-21 10:53:13.741021774 +0000 UTC m=+6.033618780 container cleanup 8199a0b362df6545a0b031557ce2948aa33d9cb647d1549c9861e25558caf68f (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-5, org.label-schema.build-date=20240731, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-09-21T10:53:13.873 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:53:13 smithi175 bash[68768]: ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-5 2024-09-21T10:53:13.873 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:13 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:53:13.706+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:30.897816+0000 front 2024-09-21T10:52:30.897736+0000 (oldest deadline 2024-09-21T10:52:54.397710+0000) 2024-09-21T10:53:13.874 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:13 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:53:13.706+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-09-21T10:52:39.698984+0000 front 2024-09-21T10:52:39.698943+0000 (oldest deadline 2024-09-21T10:53:05.598730+0000) 2024-09-21T10:53:13.874 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:13 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:53:13.706+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.123:6822 osd.2 since back 2024-09-21T10:52:45.599268+0000 front 2024-09-21T10:52:45.599206+0000 (oldest deadline 2024-09-21T10:53:10.899146+0000) 2024-09-21T10:53:14.341 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:53:13 smithi175 podman[68810]: 2024-09-21 10:53:13.992333746 +0000 UTC m=+0.720030288 container remove 8199a0b362df6545a0b031557ce2948aa33d9cb647d1549c9861e25558caf68f (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-5, org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731) 2024-09-21T10:53:14.591 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:53:14 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6[57429]: 2024-09-21T10:53:14.339+0000 7fc564743640 -1 osd.6 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:28.588077+0000 front 2024-09-21T10:52:28.588155+0000 (oldest deadline 2024-09-21T10:52:57.388747+0000) 2024-09-21T10:53:14.591 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:53:14 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6[57429]: 2024-09-21T10:53:14.339+0000 7fc564743640 -1 osd.6 174 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-09-21T10:52:40.190381+0000 front 2024-09-21T10:52:40.190460+0000 (oldest deadline 2024-09-21T10:53:02.390975+0000) 2024-09-21T10:53:14.591 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:53:14 smithi175 podman[68900]: 2024-09-21 10:53:14.342980826 +0000 UTC m=+0.101260135 container create 4affa19ac2989ef7208626a3da4a77c742b05b7076764e5acc1d49b66b44d92a (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-5-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-09-21T10:53:14.592 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:53:14 smithi175 podman[68900]: 2024-09-21 10:53:14.257633112 +0000 UTC m=+0.015912384 image pull 8f4d013db793795fef09ff5edf96e6b42ba8cd2c9158b2f0b7a5d7c1e657c063 quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472 2024-09-21T10:53:14.592 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:53:14 smithi175 podman[68900]: 2024-09-21 10:53:14.586873615 +0000 UTC m=+0.345152896 container init 4affa19ac2989ef7208626a3da4a77c742b05b7076764e5acc1d49b66b44d92a (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-5-deactivate, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-09-21T10:53:14.592 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:53:14 smithi175 podman[68900]: 2024-09-21 10:53:14.590169462 +0000 UTC m=+0.348448726 container start 4affa19ac2989ef7208626a3da4a77c742b05b7076764e5acc1d49b66b44d92a (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-5-deactivate, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True) 2024-09-21T10:53:14.873 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:14 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:53:14.674+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:30.897816+0000 front 2024-09-21T10:52:30.897736+0000 (oldest deadline 2024-09-21T10:52:54.397710+0000) 2024-09-21T10:53:14.873 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:14 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:53:14.674+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-09-21T10:52:39.698984+0000 front 2024-09-21T10:52:39.698943+0000 (oldest deadline 2024-09-21T10:53:05.598730+0000) 2024-09-21T10:53:14.873 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:14 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:53:14.674+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.123:6822 osd.2 since back 2024-09-21T10:52:45.599268+0000 front 2024-09-21T10:52:45.599206+0000 (oldest deadline 2024-09-21T10:53:10.899146+0000) 2024-09-21T10:53:14.874 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:53:14 smithi175 podman[68900]: 2024-09-21 10:53:14.703935955 +0000 UTC m=+0.462215248 container attach 4affa19ac2989ef7208626a3da4a77c742b05b7076764e5acc1d49b66b44d92a (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-5-deactivate, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3) 2024-09-21T10:53:14.874 INFO:journalctl@ceph.osd.5.smithi175.stdout:Sep 21 10:53:14 smithi175 podman[68900]: 2024-09-21 10:53:14.786611033 +0000 UTC m=+0.544890319 container died 4affa19ac2989ef7208626a3da4a77c742b05b7076764e5acc1d49b66b44d92a (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-5-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-09-21T10:53:15.377 DEBUG:teuthology.orchestra.run.smithi175:> sudo pkill -f 'journalctl -f -n 0 -u ceph-3ccd15e6-7804-11ef-baf6-efdc52797490@osd.5.service' 2024-09-21T10:53:15.416 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T10:53:15.416 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-09-21T10:53:15.416 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-09-21T10:53:15.417 DEBUG:teuthology.orchestra.run.smithi175:> sudo systemctl stop ceph-3ccd15e6-7804-11ef-baf6-efdc52797490@osd.6 2024-09-21T10:53:15.623 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:53:15 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6[57429]: 2024-09-21T10:53:15.364+0000 7fc564743640 -1 osd.6 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:28.588077+0000 front 2024-09-21T10:52:28.588155+0000 (oldest deadline 2024-09-21T10:52:57.388747+0000) 2024-09-21T10:53:15.623 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:53:15 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6[57429]: 2024-09-21T10:53:15.364+0000 7fc564743640 -1 osd.6 174 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-09-21T10:52:40.190381+0000 front 2024-09-21T10:52:40.190460+0000 (oldest deadline 2024-09-21T10:53:02.390975+0000) 2024-09-21T10:53:15.623 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:53:15 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6[57429]: 2024-09-21T10:53:15.364+0000 7fc564743640 -1 osd.6 174 heartbeat_check: no reply from 172.21.15.123:6822 osd.2 since back 2024-09-21T10:52:47.592376+0000 front 2024-09-21T10:52:47.592303+0000 (oldest deadline 2024-09-21T10:53:15.192728+0000) 2024-09-21T10:53:15.623 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:53:15 smithi175 systemd[1]: Stopping Ceph osd.6 for 3ccd15e6-7804-11ef-baf6-efdc52797490... 2024-09-21T10:53:16.123 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:15 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:53:15.723+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:30.897816+0000 front 2024-09-21T10:52:30.897736+0000 (oldest deadline 2024-09-21T10:52:54.397710+0000) 2024-09-21T10:53:16.124 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:15 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:53:15.723+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-09-21T10:52:39.698984+0000 front 2024-09-21T10:52:39.698943+0000 (oldest deadline 2024-09-21T10:53:05.598730+0000) 2024-09-21T10:53:16.124 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:15 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:53:15.723+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.123:6822 osd.2 since back 2024-09-21T10:52:45.599268+0000 front 2024-09-21T10:52:45.599206+0000 (oldest deadline 2024-09-21T10:53:10.899146+0000) 2024-09-21T10:53:16.124 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:53:15 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6[57429]: 2024-09-21T10:53:15.743+0000 7fc56c136640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:53:16.124 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:53:15 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6[57429]: 2024-09-21T10:53:15.743+0000 7fc56c136640 -1 osd.6 174 *** Got signal Terminated *** 2024-09-21T10:53:16.124 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:53:15 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6[57429]: 2024-09-21T10:53:15.743+0000 7fc56c136640 -1 osd.6 174 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T10:53:16.623 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:53:16 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6[57429]: 2024-09-21T10:53:16.324+0000 7fc564743640 -1 osd.6 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:28.588077+0000 front 2024-09-21T10:52:28.588155+0000 (oldest deadline 2024-09-21T10:52:57.388747+0000) 2024-09-21T10:53:16.624 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:53:16 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6[57429]: 2024-09-21T10:53:16.324+0000 7fc564743640 -1 osd.6 174 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-09-21T10:52:40.190381+0000 front 2024-09-21T10:52:40.190460+0000 (oldest deadline 2024-09-21T10:53:02.390975+0000) 2024-09-21T10:53:16.624 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:53:16 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6[57429]: 2024-09-21T10:53:16.324+0000 7fc564743640 -1 osd.6 174 heartbeat_check: no reply from 172.21.15.123:6822 osd.2 since back 2024-09-21T10:52:47.592376+0000 front 2024-09-21T10:52:47.592303+0000 (oldest deadline 2024-09-21T10:53:15.192728+0000) 2024-09-21T10:53:17.123 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:16 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:53:16.722+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:30.897816+0000 front 2024-09-21T10:52:30.897736+0000 (oldest deadline 2024-09-21T10:52:54.397710+0000) 2024-09-21T10:53:17.123 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:16 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:53:16.722+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-09-21T10:52:39.698984+0000 front 2024-09-21T10:52:39.698943+0000 (oldest deadline 2024-09-21T10:53:05.598730+0000) 2024-09-21T10:53:17.123 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:16 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:53:16.722+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.123:6822 osd.2 since back 2024-09-21T10:52:45.599268+0000 front 2024-09-21T10:52:45.599206+0000 (oldest deadline 2024-09-21T10:53:10.899146+0000) 2024-09-21T10:53:17.623 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:53:17 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6[57429]: 2024-09-21T10:53:17.366+0000 7fc564743640 -1 osd.6 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:28.588077+0000 front 2024-09-21T10:52:28.588155+0000 (oldest deadline 2024-09-21T10:52:57.388747+0000) 2024-09-21T10:53:17.623 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:53:17 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6[57429]: 2024-09-21T10:53:17.366+0000 7fc564743640 -1 osd.6 174 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-09-21T10:52:40.190381+0000 front 2024-09-21T10:52:40.190460+0000 (oldest deadline 2024-09-21T10:53:02.390975+0000) 2024-09-21T10:53:17.623 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:53:17 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6[57429]: 2024-09-21T10:53:17.366+0000 7fc564743640 -1 osd.6 174 heartbeat_check: no reply from 172.21.15.123:6822 osd.2 since back 2024-09-21T10:52:47.592376+0000 front 2024-09-21T10:52:47.592303+0000 (oldest deadline 2024-09-21T10:53:15.192728+0000) 2024-09-21T10:53:18.123 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:17 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:53:17.762+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:30.897816+0000 front 2024-09-21T10:52:30.897736+0000 (oldest deadline 2024-09-21T10:52:54.397710+0000) 2024-09-21T10:53:18.123 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:17 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:53:17.762+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-09-21T10:52:39.698984+0000 front 2024-09-21T10:52:39.698943+0000 (oldest deadline 2024-09-21T10:53:05.598730+0000) 2024-09-21T10:53:18.123 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:17 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:53:17.762+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.123:6822 osd.2 since back 2024-09-21T10:52:45.599268+0000 front 2024-09-21T10:52:45.599206+0000 (oldest deadline 2024-09-21T10:53:10.899146+0000) 2024-09-21T10:53:18.623 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:53:18 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6[57429]: 2024-09-21T10:53:18.327+0000 7fc564743640 -1 osd.6 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:28.588077+0000 front 2024-09-21T10:52:28.588155+0000 (oldest deadline 2024-09-21T10:52:57.388747+0000) 2024-09-21T10:53:18.623 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:53:18 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6[57429]: 2024-09-21T10:53:18.327+0000 7fc564743640 -1 osd.6 174 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-09-21T10:52:40.190381+0000 front 2024-09-21T10:52:40.190460+0000 (oldest deadline 2024-09-21T10:53:02.390975+0000) 2024-09-21T10:53:18.624 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:53:18 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6[57429]: 2024-09-21T10:53:18.327+0000 7fc564743640 -1 osd.6 174 heartbeat_check: no reply from 172.21.15.123:6822 osd.2 since back 2024-09-21T10:52:47.592376+0000 front 2024-09-21T10:52:47.592303+0000 (oldest deadline 2024-09-21T10:53:15.192728+0000) 2024-09-21T10:53:19.123 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:18 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:53:18.764+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:30.897816+0000 front 2024-09-21T10:52:30.897736+0000 (oldest deadline 2024-09-21T10:52:54.397710+0000) 2024-09-21T10:53:19.123 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:18 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:53:18.764+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-09-21T10:52:39.698984+0000 front 2024-09-21T10:52:39.698943+0000 (oldest deadline 2024-09-21T10:53:05.598730+0000) 2024-09-21T10:53:19.123 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:18 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:53:18.764+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.123:6822 osd.2 since back 2024-09-21T10:52:45.599268+0000 front 2024-09-21T10:52:45.599206+0000 (oldest deadline 2024-09-21T10:53:10.899146+0000) 2024-09-21T10:53:19.623 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:53:19 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6[57429]: 2024-09-21T10:53:19.354+0000 7fc564743640 -1 osd.6 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:28.588077+0000 front 2024-09-21T10:52:28.588155+0000 (oldest deadline 2024-09-21T10:52:57.388747+0000) 2024-09-21T10:53:19.623 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:53:19 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6[57429]: 2024-09-21T10:53:19.354+0000 7fc564743640 -1 osd.6 174 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-09-21T10:52:40.190381+0000 front 2024-09-21T10:52:40.190460+0000 (oldest deadline 2024-09-21T10:53:02.390975+0000) 2024-09-21T10:53:19.623 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:53:19 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6[57429]: 2024-09-21T10:53:19.354+0000 7fc564743640 -1 osd.6 174 heartbeat_check: no reply from 172.21.15.123:6822 osd.2 since back 2024-09-21T10:52:47.592376+0000 front 2024-09-21T10:52:47.592303+0000 (oldest deadline 2024-09-21T10:53:15.192728+0000) 2024-09-21T10:53:20.123 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:19 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:53:19.809+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:30.897816+0000 front 2024-09-21T10:52:30.897736+0000 (oldest deadline 2024-09-21T10:52:54.397710+0000) 2024-09-21T10:53:20.123 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:19 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:53:19.809+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-09-21T10:52:39.698984+0000 front 2024-09-21T10:52:39.698943+0000 (oldest deadline 2024-09-21T10:53:05.598730+0000) 2024-09-21T10:53:20.123 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:19 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:53:19.809+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.123:6822 osd.2 since back 2024-09-21T10:52:45.599268+0000 front 2024-09-21T10:52:45.599206+0000 (oldest deadline 2024-09-21T10:53:10.899146+0000) 2024-09-21T10:53:20.623 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:53:20 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6[57429]: 2024-09-21T10:53:20.345+0000 7fc564743640 -1 osd.6 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:28.588077+0000 front 2024-09-21T10:52:28.588155+0000 (oldest deadline 2024-09-21T10:52:57.388747+0000) 2024-09-21T10:53:20.623 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:53:20 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6[57429]: 2024-09-21T10:53:20.345+0000 7fc564743640 -1 osd.6 174 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-09-21T10:52:40.190381+0000 front 2024-09-21T10:52:40.190460+0000 (oldest deadline 2024-09-21T10:53:02.390975+0000) 2024-09-21T10:53:20.623 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:53:20 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6[57429]: 2024-09-21T10:53:20.345+0000 7fc564743640 -1 osd.6 174 heartbeat_check: no reply from 172.21.15.123:6822 osd.2 since back 2024-09-21T10:52:47.592376+0000 front 2024-09-21T10:52:47.592303+0000 (oldest deadline 2024-09-21T10:53:15.192728+0000) 2024-09-21T10:53:21.123 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:20 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:53:20.802+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:30.897816+0000 front 2024-09-21T10:52:30.897736+0000 (oldest deadline 2024-09-21T10:52:54.397710+0000) 2024-09-21T10:53:21.123 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:20 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:53:20.802+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-09-21T10:52:39.698984+0000 front 2024-09-21T10:52:39.698943+0000 (oldest deadline 2024-09-21T10:53:05.598730+0000) 2024-09-21T10:53:21.123 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:20 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:53:20.802+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.123:6822 osd.2 since back 2024-09-21T10:52:45.599268+0000 front 2024-09-21T10:52:45.599206+0000 (oldest deadline 2024-09-21T10:53:10.899146+0000) 2024-09-21T10:53:21.123 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:20 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:53:20.802+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.123:6830 osd.3 since back 2024-09-21T10:52:54.800953+0000 front 2024-09-21T10:52:54.800935+0000 (oldest deadline 2024-09-21T10:53:20.700762+0000) 2024-09-21T10:53:21.544 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:53:21 smithi175 podman[69001]: 2024-09-21 10:53:21.282241137 +0000 UTC m=+5.639710637 container died aab90bc08918acd17479c9d688e560d0ed82ebff8dd25a6dfbd9a2ec00fcafd7 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD) 2024-09-21T10:53:21.873 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:53:21 smithi175 podman[69001]: 2024-09-21 10:53:21.64254977 +0000 UTC m=+6.000019271 container cleanup aab90bc08918acd17479c9d688e560d0ed82ebff8dd25a6dfbd9a2ec00fcafd7 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-09-21T10:53:21.873 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:53:21 smithi175 bash[69001]: ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6 2024-09-21T10:53:21.873 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:21 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:53:21.781+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:30.897816+0000 front 2024-09-21T10:52:30.897736+0000 (oldest deadline 2024-09-21T10:52:54.397710+0000) 2024-09-21T10:53:21.873 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:21 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:53:21.781+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-09-21T10:52:39.698984+0000 front 2024-09-21T10:52:39.698943+0000 (oldest deadline 2024-09-21T10:53:05.598730+0000) 2024-09-21T10:53:21.874 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:21 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:53:21.781+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.123:6822 osd.2 since back 2024-09-21T10:52:45.599268+0000 front 2024-09-21T10:52:45.599206+0000 (oldest deadline 2024-09-21T10:53:10.899146+0000) 2024-09-21T10:53:21.874 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:21 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:53:21.781+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.123:6830 osd.3 since back 2024-09-21T10:52:54.800953+0000 front 2024-09-21T10:52:54.800935+0000 (oldest deadline 2024-09-21T10:53:20.700762+0000) 2024-09-21T10:53:22.256 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:53:21 smithi175 podman[69016]: 2024-09-21 10:53:21.926840243 +0000 UTC m=+0.643481443 container remove aab90bc08918acd17479c9d688e560d0ed82ebff8dd25a6dfbd9a2ec00fcafd7 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731) 2024-09-21T10:53:22.548 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:53:22 smithi175 podman[69105]: 2024-09-21 10:53:22.157655989 +0000 UTC m=+0.015666177 image pull 8f4d013db793795fef09ff5edf96e6b42ba8cd2c9158b2f0b7a5d7c1e657c063 quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472 2024-09-21T10:53:22.548 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:53:22 smithi175 podman[69105]: 2024-09-21 10:53:22.261145838 +0000 UTC m=+0.119156003 container create 1eed4d816f560bf733f807a76f60110a1214c80058520e6f16b68f15ba392894 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6-deactivate, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-09-21T10:53:22.548 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:53:22 smithi175 podman[69105]: 2024-09-21 10:53:22.472005206 +0000 UTC m=+0.330015386 container init 1eed4d816f560bf733f807a76f60110a1214c80058520e6f16b68f15ba392894 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6-deactivate, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-09-21T10:53:22.548 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:53:22 smithi175 podman[69105]: 2024-09-21 10:53:22.475454849 +0000 UTC m=+0.333465017 container start 1eed4d816f560bf733f807a76f60110a1214c80058520e6f16b68f15ba392894 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6-deactivate, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-09-21T10:53:22.548 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:53:22 smithi175 podman[69105]: 2024-09-21 10:53:22.546840317 +0000 UTC m=+0.404850481 container attach 1eed4d816f560bf733f807a76f60110a1214c80058520e6f16b68f15ba392894 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6-deactivate, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, ceph=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T10:53:22.832 INFO:journalctl@ceph.osd.6.smithi175.stdout:Sep 21 10:53:22 smithi175 podman[69105]: 2024-09-21 10:53:22.665109401 +0000 UTC m=+0.523119574 container died 1eed4d816f560bf733f807a76f60110a1214c80058520e6f16b68f15ba392894 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-6-deactivate, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=) 2024-09-21T10:53:23.123 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:22 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:53:22.829+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:30.897816+0000 front 2024-09-21T10:52:30.897736+0000 (oldest deadline 2024-09-21T10:52:54.397710+0000) 2024-09-21T10:53:23.123 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:22 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:53:22.829+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-09-21T10:52:39.698984+0000 front 2024-09-21T10:52:39.698943+0000 (oldest deadline 2024-09-21T10:53:05.598730+0000) 2024-09-21T10:53:23.123 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:22 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:53:22.829+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.123:6822 osd.2 since back 2024-09-21T10:52:45.599268+0000 front 2024-09-21T10:52:45.599206+0000 (oldest deadline 2024-09-21T10:53:10.899146+0000) 2024-09-21T10:53:23.124 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:22 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:53:22.829+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.123:6830 osd.3 since back 2024-09-21T10:52:54.800953+0000 front 2024-09-21T10:52:54.800935+0000 (oldest deadline 2024-09-21T10:53:20.700762+0000) 2024-09-21T10:53:23.274 DEBUG:teuthology.orchestra.run.smithi175:> sudo pkill -f 'journalctl -f -n 0 -u ceph-3ccd15e6-7804-11ef-baf6-efdc52797490@osd.6.service' 2024-09-21T10:53:23.311 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T10:53:23.311 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-09-21T10:53:23.311 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-09-21T10:53:23.311 DEBUG:teuthology.orchestra.run.smithi175:> sudo systemctl stop ceph-3ccd15e6-7804-11ef-baf6-efdc52797490@osd.7 2024-09-21T10:53:23.623 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:23 smithi175 systemd[1]: Stopping Ceph osd.7 for 3ccd15e6-7804-11ef-baf6-efdc52797490... 2024-09-21T10:53:24.123 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:23 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:53:23.653+0000 7f305febf640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:53:24.123 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:23 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:53:23.653+0000 7f305febf640 -1 osd.7 174 *** Got signal Terminated *** 2024-09-21T10:53:24.123 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:23 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:53:23.653+0000 7f305febf640 -1 osd.7 174 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T10:53:24.123 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:23 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:53:23.853+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:30.897816+0000 front 2024-09-21T10:52:30.897736+0000 (oldest deadline 2024-09-21T10:52:54.397710+0000) 2024-09-21T10:53:24.124 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:23 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:53:23.853+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-09-21T10:52:39.698984+0000 front 2024-09-21T10:52:39.698943+0000 (oldest deadline 2024-09-21T10:53:05.598730+0000) 2024-09-21T10:53:24.124 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:23 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:53:23.853+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.123:6822 osd.2 since back 2024-09-21T10:52:45.599268+0000 front 2024-09-21T10:52:45.599206+0000 (oldest deadline 2024-09-21T10:53:10.899146+0000) 2024-09-21T10:53:24.124 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:23 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:53:23.853+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.123:6830 osd.3 since back 2024-09-21T10:52:54.800953+0000 front 2024-09-21T10:52:54.800935+0000 (oldest deadline 2024-09-21T10:53:20.700762+0000) 2024-09-21T10:53:25.273 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:24 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:53:24.898+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:30.897816+0000 front 2024-09-21T10:52:30.897736+0000 (oldest deadline 2024-09-21T10:52:54.397710+0000) 2024-09-21T10:53:25.273 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:24 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:53:24.898+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-09-21T10:52:39.698984+0000 front 2024-09-21T10:52:39.698943+0000 (oldest deadline 2024-09-21T10:53:05.598730+0000) 2024-09-21T10:53:25.273 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:24 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:53:24.898+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.123:6822 osd.2 since back 2024-09-21T10:52:45.599268+0000 front 2024-09-21T10:52:45.599206+0000 (oldest deadline 2024-09-21T10:53:10.899146+0000) 2024-09-21T10:53:25.273 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:24 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:53:24.898+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.123:6830 osd.3 since back 2024-09-21T10:52:54.800953+0000 front 2024-09-21T10:52:54.800935+0000 (oldest deadline 2024-09-21T10:53:20.700762+0000) 2024-09-21T10:53:26.203 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:25 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:53:25.902+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:30.897816+0000 front 2024-09-21T10:52:30.897736+0000 (oldest deadline 2024-09-21T10:52:54.397710+0000) 2024-09-21T10:53:26.203 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:25 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:53:25.902+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-09-21T10:52:39.698984+0000 front 2024-09-21T10:52:39.698943+0000 (oldest deadline 2024-09-21T10:53:05.598730+0000) 2024-09-21T10:53:26.203 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:25 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:53:25.902+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.123:6822 osd.2 since back 2024-09-21T10:52:45.599268+0000 front 2024-09-21T10:52:45.599206+0000 (oldest deadline 2024-09-21T10:53:10.899146+0000) 2024-09-21T10:53:26.203 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:25 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:53:25.902+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.123:6830 osd.3 since back 2024-09-21T10:52:54.800953+0000 front 2024-09-21T10:52:54.800935+0000 (oldest deadline 2024-09-21T10:53:20.700762+0000) 2024-09-21T10:53:27.373 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:26 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:53:26.913+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:30.897816+0000 front 2024-09-21T10:52:30.897736+0000 (oldest deadline 2024-09-21T10:52:54.397710+0000) 2024-09-21T10:53:27.373 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:26 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:53:26.913+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-09-21T10:52:39.698984+0000 front 2024-09-21T10:52:39.698943+0000 (oldest deadline 2024-09-21T10:53:05.598730+0000) 2024-09-21T10:53:27.373 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:26 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:53:26.913+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.123:6822 osd.2 since back 2024-09-21T10:52:45.599268+0000 front 2024-09-21T10:52:45.599206+0000 (oldest deadline 2024-09-21T10:53:10.899146+0000) 2024-09-21T10:53:27.373 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:26 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:53:26.913+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.123:6830 osd.3 since back 2024-09-21T10:52:54.800953+0000 front 2024-09-21T10:52:54.800935+0000 (oldest deadline 2024-09-21T10:53:20.700762+0000) 2024-09-21T10:53:27.374 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:26 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:53:26.913+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.175:6804 osd.4 since back 2024-09-21T10:53:01.801504+0000 front 2024-09-21T10:53:01.801560+0000 (oldest deadline 2024-09-21T10:53:26.517070+0000) 2024-09-21T10:53:28.373 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:27 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:53:27.936+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.123:6806 osd.0 since back 2024-09-21T10:52:30.897816+0000 front 2024-09-21T10:52:30.897736+0000 (oldest deadline 2024-09-21T10:52:54.397710+0000) 2024-09-21T10:53:28.373 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:27 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:53:27.936+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.123:6814 osd.1 since back 2024-09-21T10:52:39.698984+0000 front 2024-09-21T10:52:39.698943+0000 (oldest deadline 2024-09-21T10:53:05.598730+0000) 2024-09-21T10:53:28.373 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:27 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:53:27.936+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.123:6822 osd.2 since back 2024-09-21T10:52:45.599268+0000 front 2024-09-21T10:52:45.599206+0000 (oldest deadline 2024-09-21T10:53:10.899146+0000) 2024-09-21T10:53:28.373 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:27 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:53:27.936+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.123:6830 osd.3 since back 2024-09-21T10:52:54.800953+0000 front 2024-09-21T10:52:54.800935+0000 (oldest deadline 2024-09-21T10:53:20.700762+0000) 2024-09-21T10:53:28.373 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:27 smithi175 ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7[59522]: 2024-09-21T10:53:27.936+0000 7f30584cc640 -1 osd.7 174 heartbeat_check: no reply from 172.21.15.175:6804 osd.4 since back 2024-09-21T10:53:01.801504+0000 front 2024-09-21T10:53:01.801560+0000 (oldest deadline 2024-09-21T10:53:26.517070+0000) 2024-09-21T10:53:29.340 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:29 smithi175 podman[69207]: 2024-09-21 10:53:29.068332204 +0000 UTC m=+5.528750116 container died 2f37c8ff343fc4f62a1d63a7013f5f1b77ab94b59eb6256adb6bc1af62e8d2d2 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, GIT_CLEAN=True, io.buildah.version=1.37.2) 2024-09-21T10:53:29.623 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:29 smithi175 podman[69207]: 2024-09-21 10:53:29.422208215 +0000 UTC m=+5.882626170 container cleanup 2f37c8ff343fc4f62a1d63a7013f5f1b77ab94b59eb6256adb6bc1af62e8d2d2 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T10:53:29.623 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:29 smithi175 bash[69207]: ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7 2024-09-21T10:53:30.083 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:29 smithi175 podman[69236]: 2024-09-21 10:53:29.742180455 +0000 UTC m=+0.662825843 container remove 2f37c8ff343fc4f62a1d63a7013f5f1b77ab94b59eb6256adb6bc1af62e8d2d2 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3) 2024-09-21T10:53:30.336 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:30 smithi175 podman[69329]: 2024-09-21 10:53:29.984477646 +0000 UTC m=+0.016000834 image pull 8f4d013db793795fef09ff5edf96e6b42ba8cd2c9158b2f0b7a5d7c1e657c063 quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472 2024-09-21T10:53:30.337 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:30 smithi175 podman[69329]: 2024-09-21 10:53:30.08623534 +0000 UTC m=+0.117758516 container create 7fdef64ca239f5bf0c9c935b555e57d959d64f458d59c459171d516860f3b016 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7-deactivate, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, org.label-schema.build-date=20240731, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-09-21T10:53:30.337 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:30 smithi175 podman[69329]: 2024-09-21 10:53:30.3319141 +0000 UTC m=+0.363437274 container init 7fdef64ca239f5bf0c9c935b555e57d959d64f458d59c459171d516860f3b016 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7-deactivate, org.label-schema.build-date=20240731, ceph=True, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T10:53:30.337 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:30 smithi175 podman[69329]: 2024-09-21 10:53:30.335413831 +0000 UTC m=+0.366937020 container start 7fdef64ca239f5bf0c9c935b555e57d959d64f458d59c459171d516860f3b016 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7-deactivate, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=) 2024-09-21T10:53:30.623 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:30 smithi175 podman[69329]: 2024-09-21 10:53:30.41506494 +0000 UTC m=+0.446588124 container attach 7fdef64ca239f5bf0c9c935b555e57d959d64f458d59c459171d516860f3b016 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7-deactivate, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T10:53:30.623 INFO:journalctl@ceph.osd.7.smithi175.stdout:Sep 21 10:53:30 smithi175 podman[69329]: 2024-09-21 10:53:30.527199744 +0000 UTC m=+0.558722931 container died 7fdef64ca239f5bf0c9c935b555e57d959d64f458d59c459171d516860f3b016 (image=quay.ceph.io/ceph-ci/ceph@sha256:3855b8c483c85027fea24c112434955a9e8e6f629214baaca7b99a8fabb55472, name=ceph-3ccd15e6-7804-11ef-baf6-efdc52797490-osd-7-deactivate, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-21-0813-5c1cdc3, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T10:53:31.193 DEBUG:teuthology.orchestra.run.smithi175:> sudo pkill -f 'journalctl -f -n 0 -u ceph-3ccd15e6-7804-11ef-baf6-efdc52797490@osd.7.service' 2024-09-21T10:53:31.235 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T10:53:31.235 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-09-21T10:53:31.236 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 --force --keep-logs 2024-09-21T10:56:25.374 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 --force --keep-logs 2024-09-21T10:56:47.058 DEBUG:teuthology.orchestra.run.smithi123:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-21T10:56:47.093 DEBUG:teuthology.orchestra.run.smithi175:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-21T10:56:47.122 INFO:tasks.cephadm:Archiving crash dumps... 2024-09-21T10:56:47.124 DEBUG:teuthology.misc:Transferring archived files from smithi123:/var/lib/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/crash to /home/teuthworker/archive/gabrioux-2024-09-21_10:07:29-orch:cephadm-wip-guits-main-2024-09-21-0813-distro-default-smithi/7913804/remote/smithi123/crash 2024-09-21T10:56:47.125 DEBUG:teuthology.orchestra.run.smithi123:> sudo tar c -f - -C /var/lib/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/crash -- . 2024-09-21T10:56:47.225 INFO:teuthology.orchestra.run.smithi123.stderr:tar: /var/lib/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/crash: Cannot open: No such file or directory 2024-09-21T10:56:47.225 INFO:teuthology.orchestra.run.smithi123.stderr:tar: Error is not recoverable: exiting now 2024-09-21T10:56:47.227 DEBUG:teuthology.misc:Transferring archived files from smithi175:/var/lib/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/crash to /home/teuthworker/archive/gabrioux-2024-09-21_10:07:29-orch:cephadm-wip-guits-main-2024-09-21-0813-distro-default-smithi/7913804/remote/smithi175/crash 2024-09-21T10:56:47.228 DEBUG:teuthology.orchestra.run.smithi175:> sudo tar c -f - -C /var/lib/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/crash -- . 2024-09-21T10:56:47.262 INFO:teuthology.orchestra.run.smithi175.stderr:tar: /var/lib/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/crash: Cannot open: No such file or directory 2024-09-21T10:56:47.262 INFO:teuthology.orchestra.run.smithi175.stderr:tar: Error is not recoverable: exiting now 2024-09-21T10:56:47.263 INFO:tasks.cephadm:Checking cluster log for badness... 2024-09-21T10:56:47.263 DEBUG:teuthology.orchestra.run.smithi123:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/ceph.log | egrep CEPHADM_ | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v CEPHADM_REFRESH_FAILED | head -n 1 2024-09-21T10:56:47.293 INFO:tasks.cephadm:Compressing logs... 2024-09-21T10:56:47.293 DEBUG:teuthology.orchestra.run.smithi123:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-21T10:56:47.336 DEBUG:teuthology.orchestra.run.smithi175:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-21T10:56:47.360 INFO:teuthology.orchestra.run.smithi175.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/ceph-osd.7.log 2024-09-21T10:56:47.361 INFO:teuthology.orchestra.run.smithi175.stderr:: No such file or directory 2024-09-21T10:56:47.361 INFO:teuthology.orchestra.run.smithi175.stderr:gzip -5 --verbose -- /var/log/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/ceph-volume.log 2024-09-21T10:56:47.361 INFO:teuthology.orchestra.run.smithi175.stderr:/var/log/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/ceph-osd.7.log: gzip -5 --verbose -- /var/log/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/ceph-osd.4.log 2024-09-21T10:56:47.361 INFO:teuthology.orchestra.run.smithi175.stderr:/var/log/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/ceph.audit.log 2024-09-21T10:56:47.366 INFO:teuthology.orchestra.run.smithi123.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/ceph-osd.0.log 2024-09-21T10:56:47.366 INFO:teuthology.orchestra.run.smithi123.stderr:: No such file or directory 2024-09-21T10:56:47.367 INFO:teuthology.orchestra.run.smithi123.stderr:gzip -5 --verbose -- /var/log/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/ceph-osd.1.log 2024-09-21T10:56:47.367 INFO:teuthology.orchestra.run.smithi123.stderr:/var/log/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/ceph-osd.0.log: gzip -5 --verbose -- /var/log/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/ceph-mon.smithi123.log 2024-09-21T10:56:47.367 INFO:teuthology.orchestra.run.smithi123.stderr:/var/log/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/ceph-volume.log 2024-09-21T10:56:47.367 INFO:teuthology.orchestra.run.smithi123.stderr:/var/log/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/ceph-mon.smithi123.log: gzip -5 --verbose -- /var/log/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/ceph.audit.log 2024-09-21T10:56:47.367 INFO:teuthology.orchestra.run.smithi123.stderr:gzip -5 --verbose --/var/log/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/ceph-volume.log: /var/log/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/ceph.cephadm.log 2024-09-21T10:56:47.367 INFO:teuthology.orchestra.run.smithi123.stderr:/var/log/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/ceph-osd.2.log 2024-09-21T10:56:47.367 INFO:teuthology.orchestra.run.smithi123.stderr:/var/log/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/ceph.log 2024-09-21T10:56:47.367 INFO:teuthology.orchestra.run.smithi123.stderr:/var/log/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/ceph-osd.2.log: gzip -5 --verbose -- /var/log/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/ceph-mgr.smithi123.stnuit.log 2024-09-21T10:56:47.367 INFO:teuthology.orchestra.run.smithi123.stderr: 85.8% -- replaced with /var/log/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/ceph.cephadm.log.gz 2024-09-21T10:56:47.367 INFO:teuthology.orchestra.run.smithi123.stderr:/var/log/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/ceph.log: gzip -5 --verbose -- /var/log/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/ceph-mds.foofs.smithi123.uqoyow.log 2024-09-21T10:56:47.368 INFO:teuthology.orchestra.run.smithi175.stderr:/var/log/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/ceph-osd.4.log: gzip -5 --verbose -- /var/log/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/ceph-mon.smithi175.log 2024-09-21T10:56:47.369 INFO:teuthology.orchestra.run.smithi175.stderr:/var/log/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/ceph.cephadm.log 2024-09-21T10:56:47.369 INFO:teuthology.orchestra.run.smithi175.stderr:/var/log/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/ceph-mon.smithi175.log: gzip -5 --verbose -- /var/log/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/ceph-mgr.smithi175.txvwrq.log 2024-09-21T10:56:47.369 INFO:teuthology.orchestra.run.smithi175.stderr:/var/log/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/ceph-osd.6.log 2024-09-21T10:56:47.369 INFO:teuthology.orchestra.run.smithi175.stderr:/var/log/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/ceph-mgr.smithi175.txvwrq.log: gzip -5 --verbose -- /var/log/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/ceph-osd.5.log 2024-09-21T10:56:47.369 INFO:teuthology.orchestra.run.smithi175.stderr: 85.8% -- replaced with /var/log/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/ceph.cephadm.log.gz 2024-09-21T10:56:47.369 INFO:teuthology.orchestra.run.smithi175.stderr:/var/log/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/ceph-osd.6.log: gzip -5 --verbose -- /var/log/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/ceph.log 2024-09-21T10:56:47.369 INFO:teuthology.orchestra.run.smithi175.stderr:/var/log/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/ceph-osd.5.log: 91.6% -- replaced with /var/log/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/ceph.audit.log.gz 2024-09-21T10:56:47.369 INFO:teuthology.orchestra.run.smithi175.stderr:gzip -5 --verbose -- /var/log/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/ceph-mds.foofs.smithi175.jrlevw.log 2024-09-21T10:56:47.370 INFO:teuthology.orchestra.run.smithi123.stderr:/var/log/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/ceph-mgr.smithi123.stnuit.log: 91.5% -- replaced with /var/log/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/ceph.audit.log.gz 2024-09-21T10:56:47.370 INFO:teuthology.orchestra.run.smithi123.stderr: 87.1% -- replaced with /var/log/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/ceph.log.gz 2024-09-21T10:56:47.370 INFO:teuthology.orchestra.run.smithi123.stderr:gzip -5 --verbose -- /var/log/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/ceph-osd.3.log 2024-09-21T10:56:47.371 INFO:teuthology.orchestra.run.smithi123.stderr:/var/log/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/ceph-mds.foofs.smithi123.uqoyow.log: gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-09-21T10:56:47.372 INFO:teuthology.orchestra.run.smithi175.stderr:/var/log/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/ceph.log: 87.1% -- replaced with /var/log/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/ceph.log.gz 2024-09-21T10:56:47.372 INFO:teuthology.orchestra.run.smithi175.stderr:gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-09-21T10:56:47.372 INFO:teuthology.orchestra.run.smithi175.stderr:/var/log/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/ceph-mds.foofs.smithi175.jrlevw.log: 88.4% -- replaced with /var/log/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/ceph-mds.foofs.smithi175.jrlevw.log.gz 2024-09-21T10:56:47.378 INFO:teuthology.orchestra.run.smithi175.stderr:/var/log/ceph/cephadm.log: 91.6% -- replaced with /var/log/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/ceph-volume.log.gz 2024-09-21T10:56:47.379 INFO:teuthology.orchestra.run.smithi175.stderr: 90.1% -- replaced with /var/log/ceph/cephadm.log.gz 2024-09-21T10:56:47.382 INFO:teuthology.orchestra.run.smithi123.stderr:/var/log/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/ceph-osd.3.log: 91.5% -- replaced with /var/log/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/ceph-volume.log.gz 2024-09-21T10:56:47.393 INFO:teuthology.orchestra.run.smithi123.stderr:/var/log/ceph/cephadm.log: 91.3% -- replaced with /var/log/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/ceph-mds.foofs.smithi123.uqoyow.log.gz 2024-09-21T10:56:47.422 INFO:teuthology.orchestra.run.smithi123.stderr: 91.8% -- replaced with /var/log/ceph/cephadm.log.gz 2024-09-21T10:56:47.432 INFO:teuthology.orchestra.run.smithi175.stderr: 89.9% -- replaced with /var/log/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/ceph-mgr.smithi175.txvwrq.log.gz 2024-09-21T10:56:47.647 INFO:teuthology.orchestra.run.smithi123.stderr: 88.7% -- replaced with /var/log/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/ceph-mgr.smithi123.stnuit.log.gz 2024-09-21T10:56:47.823 INFO:teuthology.orchestra.run.smithi175.stderr: 92.6% -- replaced with /var/log/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/ceph-mon.smithi175.log.gz 2024-09-21T10:56:48.646 INFO:teuthology.orchestra.run.smithi123.stderr: 91.9% -- replaced with /var/log/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/ceph-mon.smithi123.log.gz 2024-09-21T10:57:01.265 INFO:teuthology.orchestra.run.smithi123.stderr: 94.5% -- replaced with /var/log/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/ceph-osd.2.log.gz 2024-09-21T10:57:01.301 INFO:teuthology.orchestra.run.smithi175.stderr: 93.9% -- replaced with /var/log/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/ceph-osd.4.log.gz 2024-09-21T10:57:02.762 INFO:teuthology.orchestra.run.smithi175.stderr: 94.2% -- replaced with /var/log/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/ceph-osd.6.log.gz 2024-09-21T10:57:03.829 INFO:teuthology.orchestra.run.smithi123.stderr: 94.1% -- replaced with /var/log/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/ceph-osd.0.log.gz 2024-09-21T10:57:05.604 INFO:teuthology.orchestra.run.smithi123.stderr: 94.2% -- replaced with /var/log/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/ceph-osd.1.log.gz 2024-09-21T10:57:06.228 INFO:teuthology.orchestra.run.smithi123.stderr: 94.1% -- replaced with /var/log/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/ceph-osd.3.log.gz 2024-09-21T10:57:06.230 INFO:teuthology.orchestra.run.smithi123.stderr: 2024-09-21T10:57:06.230 INFO:teuthology.orchestra.run.smithi123.stderr:real 0m18.880s 2024-09-21T10:57:06.230 INFO:teuthology.orchestra.run.smithi123.stderr:user 1m6.347s 2024-09-21T10:57:06.230 INFO:teuthology.orchestra.run.smithi123.stderr:sys 0m2.668s 2024-09-21T10:57:07.414 INFO:teuthology.orchestra.run.smithi175.stderr: 94.2% -- replaced with /var/log/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/ceph-osd.5.log.gz 2024-09-21T10:57:09.501 INFO:teuthology.orchestra.run.smithi175.stderr: 94.1% -- replaced with /var/log/ceph/3ccd15e6-7804-11ef-baf6-efdc52797490/ceph-osd.7.log.gz 2024-09-21T10:57:09.503 INFO:teuthology.orchestra.run.smithi175.stderr: 2024-09-21T10:57:09.503 INFO:teuthology.orchestra.run.smithi175.stderr:real 0m22.152s 2024-09-21T10:57:09.503 INFO:teuthology.orchestra.run.smithi175.stderr:user 1m9.165s 2024-09-21T10:57:09.503 INFO:teuthology.orchestra.run.smithi175.stderr:sys 0m2.795s 2024-09-21T10:57:09.503 INFO:tasks.cephadm:Archiving logs... 2024-09-21T10:57:09.504 DEBUG:teuthology.misc:Transferring archived files from smithi123:/var/log/ceph to /home/teuthworker/archive/gabrioux-2024-09-21_10:07:29-orch:cephadm-wip-guits-main-2024-09-21-0813-distro-default-smithi/7913804/remote/smithi123/log 2024-09-21T10:57:09.505 DEBUG:teuthology.orchestra.run.smithi123:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-21T10:57:18.253 DEBUG:teuthology.misc:Transferring archived files from smithi175:/var/log/ceph to /home/teuthworker/archive/gabrioux-2024-09-21_10:07:29-orch:cephadm-wip-guits-main-2024-09-21-0813-distro-default-smithi/7913804/remote/smithi175/log 2024-09-21T10:57:18.254 DEBUG:teuthology.orchestra.run.smithi175:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-21T10:57:27.614 INFO:tasks.cephadm:Removing cluster... 2024-09-21T10:57:27.614 DEBUG:teuthology.orchestra.run.smithi123:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 --force 2024-09-21T10:57:27.979 DEBUG:teuthology.orchestra.run.smithi175:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 3ccd15e6-7804-11ef-baf6-efdc52797490 --force 2024-09-21T10:57:28.356 INFO:tasks.cephadm:Removing cephadm ... 2024-09-21T10:57:28.356 DEBUG:teuthology.orchestra.run.smithi123:> rm -rf /home/ubuntu/cephtest/cephadm 2024-09-21T10:57:28.373 DEBUG:teuthology.orchestra.run.smithi175:> rm -rf /home/ubuntu/cephtest/cephadm 2024-09-21T10:57:28.388 INFO:tasks.cephadm:Teardown complete 2024-09-21T10:57:28.388 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-09-21T10:57:28.397 INFO:teuthology.task.clock:Checking final clock skew... 2024-09-21T10:57:28.397 DEBUG:teuthology.orchestra.run.smithi123:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-21T10:57:28.415 DEBUG:teuthology.orchestra.run.smithi175:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-21T10:57:28.427 INFO:teuthology.orchestra.run.smithi123.stderr:bash: line 1: ntpq: command not found 2024-09-21T10:57:28.432 INFO:teuthology.orchestra.run.smithi123.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-21T10:57:28.432 INFO:teuthology.orchestra.run.smithi123.stdout:=============================================================================== 2024-09-21T10:57:28.432 INFO:teuthology.orchestra.run.smithi123.stdout:^* hv01.front.sepia.ceph.com 3 6 377 55 -140us[ -110us] +/- 53ms 2024-09-21T10:57:28.432 INFO:teuthology.orchestra.run.smithi123.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 51 +273us[ +273us] +/- 63ms 2024-09-21T10:57:28.432 INFO:teuthology.orchestra.run.smithi123.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 56 +45us[ +75us] +/- 64ms 2024-09-21T10:57:28.432 INFO:teuthology.orchestra.run.smithi123.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T10:57:28.441 INFO:teuthology.orchestra.run.smithi175.stderr:bash: line 1: ntpq: command not found 2024-09-21T10:57:28.445 INFO:teuthology.orchestra.run.smithi175.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-21T10:57:28.445 INFO:teuthology.orchestra.run.smithi175.stdout:=============================================================================== 2024-09-21T10:57:28.445 INFO:teuthology.orchestra.run.smithi175.stdout:^* hv01.front.sepia.ceph.com 3 6 377 52 -170us[ -127us] +/- 53ms 2024-09-21T10:57:28.446 INFO:teuthology.orchestra.run.smithi175.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 53 +245us[ +288us] +/- 63ms 2024-09-21T10:57:28.446 INFO:teuthology.orchestra.run.smithi175.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 53 +38us[ +81us] +/- 64ms 2024-09-21T10:57:28.446 INFO:teuthology.orchestra.run.smithi175.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T10:57:28.447 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-09-21T10:57:28.456 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-09-21T10:57:28.456 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-09-21T10:57:28.465 DEBUG:teuthology.orchestra.run.smithi123:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-21T10:57:28.550 DEBUG:teuthology.orchestra.run.smithi175:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-21T10:57:28.621 DEBUG:teuthology.orchestra.run.smithi123:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-21T10:57:28.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:57:28.650 DEBUG:teuthology.orchestra.run.smithi175:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-21T10:57:28.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:57:28.678 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-09-21T10:57:28.687 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-09-21T10:57:28.707 INFO:teuthology.task.internal:Duration was 1964.752610 seconds 2024-09-21T10:57:28.707 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-09-21T10:57:28.716 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-09-21T10:57:28.716 DEBUG:teuthology.orchestra.run.smithi123:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-21T10:57:28.718 DEBUG:teuthology.orchestra.run.smithi175:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-21T10:57:28.754 INFO:teuthology.orchestra.run.smithi123.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-21T10:57:28.757 INFO:teuthology.orchestra.run.smithi175.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-21T10:57:29.162 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-09-21T10:57:29.162 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi123.front.sepia.ceph.com 2024-09-21T10:57:29.162 DEBUG:teuthology.orchestra.run.smithi123:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-21T10:57:29.207 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi175.front.sepia.ceph.com 2024-09-21T10:57:29.207 DEBUG:teuthology.orchestra.run.smithi175:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-21T10:57:29.233 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-09-21T10:57:29.234 DEBUG:teuthology.orchestra.run.smithi123:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-21T10:57:29.249 DEBUG:teuthology.orchestra.run.smithi175:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-21T10:57:29.300 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-09-21T10:57:29.301 DEBUG:teuthology.orchestra.run.smithi123:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-21T10:57:29.315 DEBUG:teuthology.orchestra.run.smithi175:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-21T10:57:43.004 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-09-21T10:57:43.015 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-09-21T10:57:43.015 DEBUG:teuthology.orchestra.run.smithi123:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-21T10:57:43.040 DEBUG:teuthology.orchestra.run.smithi175:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-21T10:57:43.070 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-09-21T10:57:43.080 DEBUG:teuthology.orchestra.run.smithi123:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-21T10:57:43.083 DEBUG:teuthology.orchestra.run.smithi175:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-21T10:57:43.103 INFO:teuthology.orchestra.run.smithi123.stdout:kernel.core_pattern = core 2024-09-21T10:57:43.138 INFO:teuthology.orchestra.run.smithi175.stdout:kernel.core_pattern = core 2024-09-21T10:57:43.198 DEBUG:teuthology.orchestra.run.smithi123:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-21T10:57:43.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:57:43.212 DEBUG:teuthology.orchestra.run.smithi175:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-21T10:57:43.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:57:43.254 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-09-21T10:57:43.265 INFO:teuthology.task.internal:Transferring archived files... 2024-09-21T10:57:43.265 DEBUG:teuthology.misc:Transferring archived files from smithi123:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/gabrioux-2024-09-21_10:07:29-orch:cephadm-wip-guits-main-2024-09-21-0813-distro-default-smithi/7913804/remote/smithi123 2024-09-21T10:57:43.265 DEBUG:teuthology.orchestra.run.smithi123:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-21T10:57:43.324 DEBUG:teuthology.misc:Transferring archived files from smithi175:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/gabrioux-2024-09-21_10:07:29-orch:cephadm-wip-guits-main-2024-09-21-0813-distro-default-smithi/7913804/remote/smithi175 2024-09-21T10:57:43.324 DEBUG:teuthology.orchestra.run.smithi175:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-21T10:57:43.364 INFO:teuthology.task.internal:Removing archive directory... 2024-09-21T10:57:43.365 DEBUG:teuthology.orchestra.run.smithi123:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-21T10:57:43.367 DEBUG:teuthology.orchestra.run.smithi175:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-21T10:57:43.414 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-09-21T10:57:43.426 INFO:teuthology.task.internal:Not uploading archives. 2024-09-21T10:57:43.426 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-09-21T10:57:43.436 INFO:teuthology.task.internal:Tidying up after the test... 2024-09-21T10:57:43.436 DEBUG:teuthology.orchestra.run.smithi123:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-21T10:57:43.438 DEBUG:teuthology.orchestra.run.smithi175:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-21T10:57:43.451 INFO:teuthology.orchestra.run.smithi123.stdout: 265324 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 21 10:57 /home/ubuntu/cephtest 2024-09-21T10:57:43.470 INFO:teuthology.orchestra.run.smithi175.stdout: 265743 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 21 10:57 /home/ubuntu/cephtest 2024-09-21T10:57:43.471 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-09-21T10:57:43.481 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-09-21T10:57:43.502 INFO:teuthology.run:Summary data: description: orch:cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} duration: 1964.7526097297668 owner: scheduled_gabrioux@teuthology success: true 2024-09-21T10:57:43.503 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-21T10:57:43.572 INFO:teuthology.run:pass